2023-11-17T04:59:30.781 INFO:root:teuthology version: 0.0.1.dev203+g6899cd2 2023-11-17T04:59:30.782 DEBUG:teuthology.run:Teuthology command: teuthology --name yuriw-2023-11-16_22:29:26-rados-wip-yuri3-testing-2023-11-14-1227-pacific-distro-default-smithi --archive /home/teuthworker/archive/yuriw-2023-11-16_22:29:26-rados-wip-yuri3-testing-2023-11-14-1227-pacific-distro-default-smithi/7461274 --owner scheduled_yuriw@teuthology --verbose --description rados/cephadm/mds_upgrade_sequence/{bluestore-bitmap centos_8.stream_container_tools conf/{client mds mon osd} overrides/{ignorelist_health ignorelist_wrongly_marked_down pg-warn} roles tasks/{0-from/v16.2.4 1-volume/{0-create 1-ranks/2 2-allow_standby_replay/no 3-inline/yes 4-verify} 2-client 3-upgrade-with-workload 4-verify}} -- /home/teuthworker/archive/yuriw-2023-11-16_22:29:26-rados-wip-yuri3-testing-2023-11-14-1227-pacific-distro-default-smithi/7461274/orig.config.yaml 2023-11-17T04:59:30.838 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2023-11-17T04:59:30.932 INFO:teuthology.run:Config: archive_path: /home/teuthworker/archive/yuriw-2023-11-16_22:29:26-rados-wip-yuri3-testing-2023-11-14-1227-pacific-distro-default-smithi/7461274 branch: wip-yuri3-testing-2023-11-14-1227-pacific description: rados/cephadm/mds_upgrade_sequence/{bluestore-bitmap centos_8.stream_container_tools conf/{client mds mon osd} overrides/{ignorelist_health ignorelist_wrongly_marked_down pg-warn} roles tasks/{0-from/v16.2.4 1-volume/{0-create 1-ranks/2 2-allow_standby_replay/no 3-inline/yes 4-verify} 2-client 3-upgrade-with-workload 4-verify}} email: null first_in_suite: false job_id: '7461274' kernel: kdb: true sha1: distro last_in_suite: false machine_type: smithi meta: - desc: 'setup ceph/pacific v16.2.4 ' name: yuriw-2023-11-16_22:29:26-rados-wip-yuri3-testing-2023-11-14-1227-pacific-distro-default-smithi no_nested_subset: false nuke-on-error: true os_type: centos os_version: 8.stream overrides: admin_socket: branch: wip-yuri3-testing-2023-11-14-1227-pacific ceph: conf: client: client mount timeout: 600 debug client: 20 debug ms: 1 rados mon op timeout: 900 rados osd op timeout: 900 global: mon pg warn min per osd: 0 mds: debug mds: 20 debug ms: 1 mds debug frag: true mds debug scatterstat: true mds op complaint time: 180 mds verify scatter: true osd op complaint time: 180 rados mon op timeout: 900 rados osd op timeout: 900 mgr: debug mgr: 20 debug ms: 1 mon: debug mon: 20 debug ms: 1 debug paxos: 20 mon op complaint time: 120 osd: bdev async discard: true bdev enable discard: true bluestore allocator: bitmap bluestore block size: 96636764160 bluestore fsck on mount: true debug bluefs: 1/20 debug bluestore: 1/20 debug ms: 1 debug osd: 20 debug rocksdb: 4/10 mon osd backfillfull_ratio: 0.85 mon osd full ratio: 0.9 mon osd nearfull ratio: 0.8 osd failsafe full ratio: 0.95 osd objectstore: bluestore osd op complaint time: 180 flavor: default fs: xfs log-ignorelist: - \(MDS_ALL_DOWN\) - \(MDS_UP_LESS_THAN_MAX\) - overall HEALTH_ - \(FS_DEGRADED\) - \(MDS_FAILED\) - \(MDS_DEGRADED\) - \(FS_WITH_FAILED_MDS\) - \(MDS_DAMAGE\) - \(MDS_ALL_DOWN\) - \(MDS_UP_LESS_THAN_MAX\) - \(FS_INLINE_DATA_DEPRECATED\) - overall HEALTH_ - \(OSD_DOWN\) - \(OSD_ - but it is still running - is not responding sha1: ed3c8e926b9de93026c8a7c1d86be5d6821b52ac ceph-deploy: bluestore: true conf: client: log file: /var/log/ceph/ceph-$name.$pid.log mon: {} osd: bdev async discard: true bdev enable discard: true bluestore block size: 96636764160 bluestore fsck on mount: true debug bluefs: 1/20 debug bluestore: 1/20 debug rocksdb: 4/10 mon osd backfillfull_ratio: 0.85 mon osd full ratio: 0.9 mon osd nearfull ratio: 0.8 osd failsafe full ratio: 0.95 osd objectstore: bluestore fs: xfs install: ceph: flavor: default sha1: ed3c8e926b9de93026c8a7c1d86be5d6821b52ac selinux: whitelist: - scontext=system_u:system_r:logrotate_t:s0 thrashosds: bdev_inject_crash: 2 bdev_inject_crash_probability: 0.5 workunit: branch: wip-yuri3-testing-2023-11-14-1227-pacific sha1: ed3c8e926b9de93026c8a7c1d86be5d6821b52ac owner: scheduled_yuriw@teuthology priority: 75 repo: https://github.com/ceph/ceph-ci.git roles: - - host.a - client.0 - osd.0 - osd.1 - osd.2 - - host.b - client.1 - osd.3 - osd.4 - osd.5 seed: 7739 sha1: ed3c8e926b9de93026c8a7c1d86be5d6821b52ac sleep_before_teardown: 0 subset: 111/120000 suite: rados suite_branch: wip-yuri3-testing-2023-11-14-1227-pacific suite_path: /home/teuthworker/src/github.com_ceph_ceph-c_ed3c8e926b9de93026c8a7c1d86be5d6821b52ac/qa suite_relpath: qa suite_repo: https://github.com/ceph/ceph-ci.git suite_sha1: ed3c8e926b9de93026c8a7c1d86be5d6821b52ac targets: smithi040.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBKtHuG7UjUvnBw2JtooGL3kFkXlILUs0Oz0JYpqOe5VKp5gQo30S3BAyRJbfQGXgp1jj8Z4Ql6GUm7/BV4/gpLo= smithi066.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBLuHCVetiuoY465BMAkkaNSxMexMeQjvFwnGnetv25pgKW80KHb0O0XeMxOl8GRkSeP084DKy25hDE6z+CucA4E= tasks: - pexec: all: - sudo cp /etc/containers/registries.conf /etc/containers/registries.conf.backup - sudo dnf -y module reset container-tools - sudo dnf -y module install container-tools --allowerasing --nobest - sudo cp /etc/containers/registries.conf.backup /etc/containers/registries.conf - pexec: clients: - sudo modprobe -r ceph - sudo modprobe ceph disable_send_metrics=on - install: exclude_packages: - ceph-volume tag: v16.2.4 - print: '**** done install task...' - cephadm: avoid_pacific_features: true cephadm_branch: v16.2.4 cephadm_git_url: https://github.com/ceph/ceph image: docker.io/ceph/ceph:v16.2.4 roleless: true - print: '**** done starting v16.2.4' - cephadm.shell: host.a: - ceph orch status - ceph orch ps - ceph orch ls - ceph orch host ls - ceph orch device ls - cephadm.shell: host.a: - ceph fs volume create cephfs --placement=4 - ceph fs dump - cephadm.shell: host.a: - ceph fs set cephfs max_mds 2 - cephadm.shell: host.a: - ceph fs set cephfs allow_standby_replay false - cephadm.shell: host.a: - ceph fs set cephfs inline_data true --yes-i-really-really-mean-it - cephadm.shell: host.a: - ceph fs dump - ceph --format=json fs dump | jq -e ".filesystems | length == 1" - while ! ceph --format=json mds versions | jq -e ". | add == 4"; do sleep 1; done - fs.pre_upgrade_save: null - kclient: null - print: '**** done client' - parallel: - upgrade-tasks - workload-tasks - cephadm.shell: host.a: - ceph fs dump - fs.post_upgrade_checks: null teuthology: fragments_dropped: [] meta: {} postmerge: [] teuthology_branch: main teuthology_sha1: 6899cd26fceddb2fec83dc1a1349394b28c8998e timestamp: 2023-11-16_22:29:26 tube: smithi upgrade-tasks: sequential: - cephadm.shell: env: - sha1 host.a: - ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force - ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force - ceph config set global log_to_journald false --force - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 - cephadm.shell: env: - sha1 host.a: - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph fs dump; ceph orch upgrade status ; sleep 30 ; done - ceph orch ps - ceph versions - echo "wait for servicemap items w/ changing names to refresh" - sleep 60 - ceph orch ps - ceph versions - ceph versions | jq -e '.overall | length == 1' - ceph versions | jq -e '.overall | keys' | grep $sha1 user: yuriw verbose: true worker_log: /home/teuthworker/archive/worker_logs/dispatcher.smithi.1224587 workload-tasks: sequential: - workunit: clients: all: - suites/fsstress.sh 2023-11-17T04:59:30.932 INFO:teuthology.run:suite_path is set to /home/teuthworker/src/github.com_ceph_ceph-c_ed3c8e926b9de93026c8a7c1d86be5d6821b52ac/qa; will attempt to use it 2023-11-17T04:59:30.933 INFO:teuthology.run:Found tasks at /home/teuthworker/src/github.com_ceph_ceph-c_ed3c8e926b9de93026c8a7c1d86be5d6821b52ac/qa/tasks 2023-11-17T04:59:30.933 INFO:teuthology.run_tasks:Running task internal.check_packages... 2023-11-17T04:59:30.936 INFO:teuthology.task.internal:Checking packages... 2023-11-17T04:59:30.965 INFO:teuthology.task.internal:Checking packages for os_type 'centos', flavor 'default' and ceph hash 'ed3c8e926b9de93026c8a7c1d86be5d6821b52ac' 2023-11-17T04:59:30.965 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2023-11-17T04:59:30.965 INFO:teuthology.packaging:ref: None 2023-11-17T04:59:30.965 INFO:teuthology.packaging:tag: None 2023-11-17T04:59:30.965 INFO:teuthology.packaging:branch: wip-yuri3-testing-2023-11-14-1227-pacific 2023-11-17T04:59:30.965 INFO:teuthology.packaging:sha1: ed3c8e926b9de93026c8a7c1d86be5d6821b52ac 2023-11-17T04:59:30.965 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F8%2Fx86_64&ref=wip-yuri3-testing-2023-11-14-1227-pacific 2023-11-17T04:59:31.125 INFO:teuthology.task.internal:Found packages for ceph version 16.2.14-171.ged3c8e92 2023-11-17T04:59:31.125 INFO:teuthology.run_tasks:Running task internal.buildpackages_prep... 2023-11-17T04:59:31.132 INFO:teuthology.task.internal:no buildpackages task found 2023-11-17T04:59:31.132 INFO:teuthology.run_tasks:Running task internal.save_config... 2023-11-17T04:59:31.170 INFO:teuthology.task.internal:Saving configuration 2023-11-17T04:59:31.188 INFO:teuthology.run_tasks:Running task internal.check_lock... 2023-11-17T04:59:31.211 INFO:teuthology.task.internal.check_lock:Checking locks... 2023-11-17T04:59:31.237 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi040.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/yuriw-2023-11-16_22:29:26-rados-wip-yuri3-testing-2023-11-14-1227-pacific-distro-default-smithi/7461274', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '8', 'arch': 'x86_64', 'locked': True, 'locked_since': '2023-11-17 04:52:24.161988', 'locked_by': 'scheduled_yuriw@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBKtHuG7UjUvnBw2JtooGL3kFkXlILUs0Oz0JYpqOe5VKp5gQo30S3BAyRJbfQGXgp1jj8Z4Ql6GUm7/BV4/gpLo='} 2023-11-17T04:59:31.264 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi066.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/yuriw-2023-11-16_22:29:26-rados-wip-yuri3-testing-2023-11-14-1227-pacific-distro-default-smithi/7461274', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '8', 'arch': 'x86_64', 'locked': True, 'locked_since': '2023-11-17 04:52:24.163889', 'locked_by': 'scheduled_yuriw@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBLuHCVetiuoY465BMAkkaNSxMexMeQjvFwnGnetv25pgKW80KHb0O0XeMxOl8GRkSeP084DKy25hDE6z+CucA4E='} 2023-11-17T04:59:31.265 INFO:teuthology.run_tasks:Running task internal.add_remotes... 2023-11-17T04:59:31.271 INFO:teuthology.task.internal:roles: ubuntu@smithi040.front.sepia.ceph.com - ['host.a', 'client.0', 'osd.0', 'osd.1', 'osd.2'] 2023-11-17T04:59:31.271 INFO:teuthology.task.internal:roles: ubuntu@smithi066.front.sepia.ceph.com - ['host.b', 'client.1', 'osd.3', 'osd.4', 'osd.5'] 2023-11-17T04:59:31.271 INFO:teuthology.run_tasks:Running task console_log... 2023-11-17T04:59:31.382 DEBUG:teuthology.exit:Installing handler: Handler(exiter=, func=.kill_console_loggers at 0x7fdc0292bf70>, signals=[15]) 2023-11-17T04:59:31.383 INFO:teuthology.run_tasks:Running task internal.connect... 2023-11-17T04:59:31.388 INFO:teuthology.task.internal:Opening connections... 2023-11-17T04:59:31.388 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi040.front.sepia.ceph.com 2023-11-17T04:59:31.390 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi040.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-11-17T04:59:31.506 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi066.front.sepia.ceph.com 2023-11-17T04:59:31.508 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi066.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-11-17T04:59:31.575 INFO:teuthology.run_tasks:Running task internal.push_inventory... 2023-11-17T04:59:31.583 DEBUG:teuthology.orchestra.run.smithi040:> uname -m 2023-11-17T04:59:31.607 INFO:teuthology.orchestra.run.smithi040.stdout:x86_64 2023-11-17T04:59:31.607 DEBUG:teuthology.orchestra.run.smithi040:> cat /etc/os-release 2023-11-17T04:59:31.671 INFO:teuthology.orchestra.run.smithi040.stdout:NAME="CentOS Stream" 2023-11-17T04:59:31.671 INFO:teuthology.orchestra.run.smithi040.stdout:VERSION="8" 2023-11-17T04:59:31.671 INFO:teuthology.orchestra.run.smithi040.stdout:ID="centos" 2023-11-17T04:59:31.672 INFO:teuthology.orchestra.run.smithi040.stdout:ID_LIKE="rhel fedora" 2023-11-17T04:59:31.673 INFO:teuthology.orchestra.run.smithi040.stdout:VERSION_ID="8" 2023-11-17T04:59:31.673 INFO:teuthology.orchestra.run.smithi040.stdout:PLATFORM_ID="platform:el8" 2023-11-17T04:59:31.673 INFO:teuthology.orchestra.run.smithi040.stdout:PRETTY_NAME="CentOS Stream 8" 2023-11-17T04:59:31.673 INFO:teuthology.orchestra.run.smithi040.stdout:ANSI_COLOR="0;31" 2023-11-17T04:59:31.673 INFO:teuthology.orchestra.run.smithi040.stdout:CPE_NAME="cpe:/o:centos:centos:8" 2023-11-17T04:59:31.676 INFO:teuthology.orchestra.run.smithi040.stdout:HOME_URL="https://centos.org/" 2023-11-17T04:59:31.676 INFO:teuthology.orchestra.run.smithi040.stdout:BUG_REPORT_URL="https://bugzilla.redhat.com/" 2023-11-17T04:59:31.677 INFO:teuthology.orchestra.run.smithi040.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 8" 2023-11-17T04:59:31.677 INFO:teuthology.orchestra.run.smithi040.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream" 2023-11-17T04:59:31.680 INFO:teuthology.lock.ops:Updating smithi040.front.sepia.ceph.com on lock server 2023-11-17T04:59:31.705 DEBUG:teuthology.orchestra.run.smithi066:> uname -m 2023-11-17T04:59:31.728 INFO:teuthology.orchestra.run.smithi066.stdout:x86_64 2023-11-17T04:59:31.728 DEBUG:teuthology.orchestra.run.smithi066:> cat /etc/os-release 2023-11-17T04:59:31.790 INFO:teuthology.orchestra.run.smithi066.stdout:NAME="CentOS Stream" 2023-11-17T04:59:31.790 INFO:teuthology.orchestra.run.smithi066.stdout:VERSION="8" 2023-11-17T04:59:31.790 INFO:teuthology.orchestra.run.smithi066.stdout:ID="centos" 2023-11-17T04:59:31.791 INFO:teuthology.orchestra.run.smithi066.stdout:ID_LIKE="rhel fedora" 2023-11-17T04:59:31.791 INFO:teuthology.orchestra.run.smithi066.stdout:VERSION_ID="8" 2023-11-17T04:59:31.791 INFO:teuthology.orchestra.run.smithi066.stdout:PLATFORM_ID="platform:el8" 2023-11-17T04:59:31.791 INFO:teuthology.orchestra.run.smithi066.stdout:PRETTY_NAME="CentOS Stream 8" 2023-11-17T04:59:31.791 INFO:teuthology.orchestra.run.smithi066.stdout:ANSI_COLOR="0;31" 2023-11-17T04:59:31.791 INFO:teuthology.orchestra.run.smithi066.stdout:CPE_NAME="cpe:/o:centos:centos:8" 2023-11-17T04:59:31.791 INFO:teuthology.orchestra.run.smithi066.stdout:HOME_URL="https://centos.org/" 2023-11-17T04:59:31.791 INFO:teuthology.orchestra.run.smithi066.stdout:BUG_REPORT_URL="https://bugzilla.redhat.com/" 2023-11-17T04:59:31.792 INFO:teuthology.orchestra.run.smithi066.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 8" 2023-11-17T04:59:31.792 INFO:teuthology.orchestra.run.smithi066.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream" 2023-11-17T04:59:31.792 INFO:teuthology.lock.ops:Updating smithi066.front.sepia.ceph.com on lock server 2023-11-17T04:59:31.812 INFO:teuthology.run_tasks:Running task internal.serialize_remote_roles... 2023-11-17T04:59:31.821 INFO:teuthology.run_tasks:Running task internal.check_conflict... 2023-11-17T04:59:31.852 INFO:teuthology.task.internal:Checking for old test directory... 2023-11-17T04:59:31.852 DEBUG:teuthology.orchestra.run.smithi040:> test '!' -e /home/ubuntu/cephtest 2023-11-17T04:59:31.855 DEBUG:teuthology.orchestra.run.smithi066:> test '!' -e /home/ubuntu/cephtest 2023-11-17T04:59:31.875 INFO:teuthology.run_tasks:Running task internal.check_ceph_data... 2023-11-17T04:59:31.886 INFO:teuthology.task.internal:Checking for non-empty /var/lib/ceph... 2023-11-17T04:59:31.886 DEBUG:teuthology.orchestra.run.smithi040:> test -z $(ls -A /var/lib/ceph) 2023-11-17T04:59:31.917 DEBUG:teuthology.orchestra.run.smithi066:> test -z $(ls -A /var/lib/ceph) 2023-11-17T04:59:31.951 INFO:teuthology.run_tasks:Running task internal.vm_setup... 2023-11-17T04:59:32.006 INFO:teuthology.run_tasks:Running task kernel... 2023-11-17T04:59:32.029 INFO:teuthology.task.kernel:normalize config orig: {'kdb': True, 'sha1': 'distro'} 2023-11-17T04:59:32.029 INFO:teuthology.task.kernel:config {'host.a': {'kdb': True, 'sha1': 'distro'}, 'host.b': {'kdb': True, 'sha1': 'distro'}}, timeout 300 2023-11-17T04:59:32.029 DEBUG:teuthology.orchestra.run.smithi040:> test -f /run/.containerenv -o -f /.dockerenv 2023-11-17T04:59:32.030 DEBUG:teuthology.orchestra.run.smithi066:> test -f /run/.containerenv -o -f /.dockerenv 2023-11-17T04:59:32.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-11-17T04:59:32.050 DEBUG:teuthology.orchestra.run.smithi040:> uname -r 2023-11-17T04:59:32.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-11-17T04:59:32.051 DEBUG:teuthology.orchestra.run.smithi066:> uname -r 2023-11-17T04:59:32.111 INFO:teuthology.orchestra.run.smithi040.stdout:4.18.0-489.el8.x86_64 2023-11-17T04:59:32.139 INFO:teuthology.task.kernel:Running kernel on smithi040: 4.18.0-489.el8.x86_64 2023-11-17T04:59:32.139 DEBUG:teuthology.orchestra.run.smithi040:> sudo yum install -y kernel 2023-11-17T04:59:32.140 INFO:teuthology.orchestra.run.smithi066.stdout:4.18.0-489.el8.x86_64 2023-11-17T04:59:32.141 INFO:teuthology.task.kernel:Running kernel on smithi066: 4.18.0-489.el8.x86_64 2023-11-17T04:59:32.141 DEBUG:teuthology.orchestra.run.smithi066:> sudo yum install -y kernel 2023-11-17T04:59:42.992 INFO:teuthology.orchestra.run.smithi040.stdout:CentOS Stream 8 - AppStream 4.1 MB/s | 34 MB 00:08 2023-11-17T04:59:50.777 INFO:teuthology.orchestra.run.smithi066.stdout:CentOS Stream 8 - AppStream 2.1 MB/s | 34 MB 00:15 2023-11-17T04:59:51.917 INFO:teuthology.orchestra.run.smithi040.stdout:CentOS Stream 8 - BaseOS 24 MB/s | 53 MB 00:02 2023-11-17T04:59:59.735 INFO:teuthology.orchestra.run.smithi066.stdout:CentOS Stream 8 - BaseOS 22 MB/s | 53 MB 00:02 2023-11-17T05:00:00.383 INFO:teuthology.orchestra.run.smithi040.stdout:CentOS Stream 8 - Extras 55 kB/s | 18 kB 00:00 2023-11-17T05:00:00.851 INFO:teuthology.orchestra.run.smithi040.stdout:CentOS Stream 8 - Extras common packages 25 kB/s | 6.9 kB 00:00 2023-11-17T05:00:01.152 INFO:teuthology.orchestra.run.smithi040.stdout:Copr repo for python3-asyncssh owned by ceph 25 kB/s | 3.5 kB 00:00 2023-11-17T05:00:02.066 INFO:teuthology.orchestra.run.smithi040.stdout:Extra Packages for Enterprise Linux 21 MB/s | 16 MB 00:00 2023-11-17T05:00:06.153 INFO:teuthology.orchestra.run.smithi040.stdout:lab-extras 310 kB/s | 24 kB 00:00 2023-11-17T05:00:10.019 INFO:teuthology.orchestra.run.smithi040.stdout:Package kernel-4.18.0-348.el8.x86_64 is already installed. 2023-11-17T05:00:10.019 INFO:teuthology.orchestra.run.smithi040.stdout:Package kernel-4.18.0-489.el8.x86_64 is already installed. 2023-11-17T05:00:10.094 INFO:teuthology.orchestra.run.smithi066.stdout:CentOS Stream 8 - Extras 7.5 kB/s | 18 kB 00:02 2023-11-17T05:00:10.419 INFO:teuthology.orchestra.run.smithi040.stdout:Dependencies resolved. 2023-11-17T05:00:10.420 INFO:teuthology.orchestra.run.smithi040.stdout:================================================================================ 2023-11-17T05:00:10.420 INFO:teuthology.orchestra.run.smithi040.stdout: Package Architecture Version Repository Size 2023-11-17T05:00:10.420 INFO:teuthology.orchestra.run.smithi040.stdout:================================================================================ 2023-11-17T05:00:10.420 INFO:teuthology.orchestra.run.smithi040.stdout:Installing: 2023-11-17T05:00:10.421 INFO:teuthology.orchestra.run.smithi040.stdout: kernel x86_64 4.18.0-521.el8 baseos 10 M 2023-11-17T05:00:10.421 INFO:teuthology.orchestra.run.smithi040.stdout:Installing dependencies: 2023-11-17T05:00:10.421 INFO:teuthology.orchestra.run.smithi040.stdout: kernel-core x86_64 4.18.0-521.el8 baseos 43 M 2023-11-17T05:00:10.421 INFO:teuthology.orchestra.run.smithi040.stdout: kernel-modules x86_64 4.18.0-521.el8 baseos 36 M 2023-11-17T05:00:10.421 INFO:teuthology.orchestra.run.smithi040.stdout: 2023-11-17T05:00:10.421 INFO:teuthology.orchestra.run.smithi040.stdout:Transaction Summary 2023-11-17T05:00:10.421 INFO:teuthology.orchestra.run.smithi040.stdout:================================================================================ 2023-11-17T05:00:10.421 INFO:teuthology.orchestra.run.smithi040.stdout:Install 3 Packages 2023-11-17T05:00:10.421 INFO:teuthology.orchestra.run.smithi040.stdout: 2023-11-17T05:00:10.422 INFO:teuthology.orchestra.run.smithi040.stdout:Total download size: 89 M 2023-11-17T05:00:10.422 INFO:teuthology.orchestra.run.smithi040.stdout:Installed size: 96 M 2023-11-17T05:00:10.422 INFO:teuthology.orchestra.run.smithi040.stdout:Downloading Packages: 2023-11-17T05:00:14.935 INFO:teuthology.orchestra.run.smithi066.stdout:CentOS Stream 8 - Extras common packages 1.5 kB/s | 6.9 kB 00:04 2023-11-17T05:00:15.223 INFO:teuthology.orchestra.run.smithi066.stdout:Copr repo for python3-asyncssh owned by ceph 30 kB/s | 3.5 kB 00:00 2023-11-17T05:00:16.198 INFO:teuthology.orchestra.run.smithi066.stdout:Extra Packages for Enterprise Linux 20 MB/s | 16 MB 00:00 2023-11-17T05:00:20.296 INFO:teuthology.orchestra.run.smithi066.stdout:lab-extras 488 kB/s | 24 kB 00:00 2023-11-17T05:00:24.168 INFO:teuthology.orchestra.run.smithi066.stdout:Package kernel-4.18.0-348.el8.x86_64 is already installed. 2023-11-17T05:00:24.168 INFO:teuthology.orchestra.run.smithi066.stdout:Package kernel-4.18.0-489.el8.x86_64 is already installed. 2023-11-17T05:00:24.466 INFO:teuthology.orchestra.run.smithi066.stdout:Dependencies resolved. 2023-11-17T05:00:24.467 INFO:teuthology.orchestra.run.smithi066.stdout:================================================================================ 2023-11-17T05:00:24.467 INFO:teuthology.orchestra.run.smithi066.stdout: Package Architecture Version Repository Size 2023-11-17T05:00:24.467 INFO:teuthology.orchestra.run.smithi066.stdout:================================================================================ 2023-11-17T05:00:24.467 INFO:teuthology.orchestra.run.smithi066.stdout:Installing: 2023-11-17T05:00:24.468 INFO:teuthology.orchestra.run.smithi066.stdout: kernel x86_64 4.18.0-521.el8 baseos 10 M 2023-11-17T05:00:24.468 INFO:teuthology.orchestra.run.smithi066.stdout:Installing dependencies: 2023-11-17T05:00:24.468 INFO:teuthology.orchestra.run.smithi066.stdout: kernel-core x86_64 4.18.0-521.el8 baseos 43 M 2023-11-17T05:00:24.468 INFO:teuthology.orchestra.run.smithi066.stdout: kernel-modules x86_64 4.18.0-521.el8 baseos 36 M 2023-11-17T05:00:24.468 INFO:teuthology.orchestra.run.smithi066.stdout: 2023-11-17T05:00:24.468 INFO:teuthology.orchestra.run.smithi066.stdout:Transaction Summary 2023-11-17T05:00:24.468 INFO:teuthology.orchestra.run.smithi066.stdout:================================================================================ 2023-11-17T05:00:24.468 INFO:teuthology.orchestra.run.smithi066.stdout:Install 3 Packages 2023-11-17T05:00:24.468 INFO:teuthology.orchestra.run.smithi066.stdout: 2023-11-17T05:00:24.469 INFO:teuthology.orchestra.run.smithi066.stdout:Total download size: 89 M 2023-11-17T05:00:24.469 INFO:teuthology.orchestra.run.smithi066.stdout:Installed size: 96 M 2023-11-17T05:00:24.469 INFO:teuthology.orchestra.run.smithi066.stdout:Downloading Packages: 2023-11-17T05:00:27.289 INFO:teuthology.orchestra.run.smithi066.stdout:(1/3): kernel-4.18.0-521.el8.x86_64.rpm 8.6 MB/s | 10 MB 00:01 2023-11-17T05:00:31.261 INFO:teuthology.orchestra.run.smithi040.stdout:(1/3): kernel-4.18.0-521.el8.x86_64.rpm 8.4 MB/s | 10 MB 00:01 2023-11-17T05:00:31.261 INFO:teuthology.orchestra.run.smithi040.stdout:(2/3): kernel-modules-4.18.0-521.el8.x86_64.rpm 16 MB/s | 36 MB 00:02 2023-11-17T05:00:31.261 INFO:teuthology.orchestra.run.smithi040.stdout:(3/3): kernel-core-4.18.0-521.el8.x86_64.rpm 15 MB/s | 43 MB 00:02 2023-11-17T05:00:31.262 INFO:teuthology.orchestra.run.smithi040.stdout:-------------------------------------------------------------------------------- 2023-11-17T05:00:31.262 INFO:teuthology.orchestra.run.smithi040.stdout:Total 4.6 MB/s | 89 MB 00:19 2023-11-17T05:00:31.262 INFO:teuthology.orchestra.run.smithi040.stdout:Running transaction check 2023-11-17T05:00:31.262 INFO:teuthology.orchestra.run.smithi040.stdout:Transaction check succeeded. 2023-11-17T05:00:31.262 INFO:teuthology.orchestra.run.smithi040.stdout:Running transaction test 2023-11-17T05:00:31.262 INFO:teuthology.orchestra.run.smithi066.stdout:(2/3): kernel-modules-4.18.0-521.el8.x86_64.rpm 16 MB/s | 36 MB 00:02 2023-11-17T05:00:31.262 INFO:teuthology.orchestra.run.smithi066.stdout:(3/3): kernel-core-4.18.0-521.el8.x86_64.rpm 17 MB/s | 43 MB 00:02 2023-11-17T05:00:31.262 INFO:teuthology.orchestra.run.smithi066.stdout:-------------------------------------------------------------------------------- 2023-11-17T05:00:31.263 INFO:teuthology.orchestra.run.smithi066.stdout:Total 21 MB/s | 89 MB 00:04 2023-11-17T05:00:31.263 INFO:teuthology.orchestra.run.smithi066.stdout:Running transaction check 2023-11-17T05:00:31.263 INFO:teuthology.orchestra.run.smithi066.stdout:Transaction check succeeded. 2023-11-17T05:00:31.263 INFO:teuthology.orchestra.run.smithi066.stdout:Running transaction test 2023-11-17T05:00:31.263 INFO:teuthology.orchestra.run.smithi066.stdout:Transaction test succeeded. 2023-11-17T05:00:31.263 INFO:teuthology.orchestra.run.smithi066.stdout:Running transaction 2023-11-17T05:00:32.446 INFO:teuthology.orchestra.run.smithi040.stdout:Transaction test succeeded. 2023-11-17T05:00:32.477 INFO:teuthology.orchestra.run.smithi040.stdout:Running transaction 2023-11-17T05:00:34.075 INFO:teuthology.orchestra.run.smithi066.stdout: Preparing : 1/1 2023-11-17T05:00:34.595 INFO:teuthology.orchestra.run.smithi066.stdout: Installing : kernel-core-4.18.0-521.el8.x86_64 1/3 2023-11-17T05:00:35.377 INFO:teuthology.orchestra.run.smithi040.stdout: Preparing : 1/1 2023-11-17T05:00:35.912 INFO:teuthology.orchestra.run.smithi040.stdout: Installing : kernel-core-4.18.0-521.el8.x86_64 1/3 2023-11-17T05:00:38.359 INFO:teuthology.orchestra.run.smithi066.stdout: Running scriptlet: kernel-core-4.18.0-521.el8.x86_64 1/3 2023-11-17T05:00:38.770 INFO:teuthology.orchestra.run.smithi066.stdout: Installing : kernel-modules-4.18.0-521.el8.x86_64 2/3 2023-11-17T05:00:39.647 INFO:teuthology.orchestra.run.smithi040.stdout: Running scriptlet: kernel-core-4.18.0-521.el8.x86_64 1/3 2023-11-17T05:00:40.036 INFO:teuthology.orchestra.run.smithi040.stdout: Installing : kernel-modules-4.18.0-521.el8.x86_64 2/3 2023-11-17T05:00:42.444 INFO:teuthology.orchestra.run.smithi066.stdout: Running scriptlet: kernel-modules-4.18.0-521.el8.x86_64 2/3 2023-11-17T05:00:42.711 INFO:teuthology.orchestra.run.smithi066.stdout: Installing : kernel-4.18.0-521.el8.x86_64 3/3 2023-11-17T05:00:43.707 INFO:teuthology.orchestra.run.smithi040.stdout: Running scriptlet: kernel-modules-4.18.0-521.el8.x86_64 2/3 2023-11-17T05:00:43.970 INFO:teuthology.orchestra.run.smithi040.stdout: Installing : kernel-4.18.0-521.el8.x86_64 3/3 2023-11-17T05:01:05.869 INFO:teuthology.orchestra.run.smithi066.stdout: Running scriptlet: kernel-core-4.18.0-521.el8.x86_64 3/3 2023-11-17T05:01:05.893 INFO:teuthology.orchestra.run.smithi066.stdout: Running scriptlet: kernel-modules-4.18.0-521.el8.x86_64 3/3 2023-11-17T05:01:06.256 INFO:teuthology.orchestra.run.smithi066.stdout: Running scriptlet: kernel-4.18.0-521.el8.x86_64 3/3 2023-11-17T05:01:06.256 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : kernel-4.18.0-521.el8.x86_64 1/3 2023-11-17T05:01:06.256 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : kernel-core-4.18.0-521.el8.x86_64 2/3 2023-11-17T05:01:07.209 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : kernel-modules-4.18.0-521.el8.x86_64 3/3 2023-11-17T05:01:07.209 INFO:teuthology.orchestra.run.smithi066.stdout: 2023-11-17T05:01:07.209 INFO:teuthology.orchestra.run.smithi066.stdout:Installed: 2023-11-17T05:01:07.209 INFO:teuthology.orchestra.run.smithi066.stdout: kernel-4.18.0-521.el8.x86_64 kernel-core-4.18.0-521.el8.x86_64 2023-11-17T05:01:07.209 INFO:teuthology.orchestra.run.smithi066.stdout: kernel-modules-4.18.0-521.el8.x86_64 2023-11-17T05:01:07.209 INFO:teuthology.orchestra.run.smithi066.stdout: 2023-11-17T05:01:07.209 INFO:teuthology.orchestra.run.smithi066.stdout:Complete! 2023-11-17T05:01:07.333 INFO:teuthology.orchestra.run.smithi040.stdout: Running scriptlet: kernel-core-4.18.0-521.el8.x86_64 3/3 2023-11-17T05:01:07.358 INFO:teuthology.orchestra.run.smithi040.stdout: Running scriptlet: kernel-modules-4.18.0-521.el8.x86_64 3/3 2023-11-17T05:01:07.373 DEBUG:teuthology.orchestra.run.smithi066:> rpm -q kernel | sort -rV | head -n 1 2023-11-17T05:01:07.609 INFO:teuthology.orchestra.run.smithi066.stdout:kernel-4.18.0-521.el8.x86_64 2023-11-17T05:01:07.609 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 4.18.0-521.el8.x86_64 2023-11-17T05:01:07.609 INFO:teuthology.task.kernel:Not newest distro kernel. Current: 4.18.0-489.el8.x86_64 Expected: 4.18.0-521.el8.x86_64 2023-11-17T05:01:07.609 INFO:teuthology.task.kernel:Skipping firmware on distro kernel 2023-11-17T05:01:07.609 DEBUG:teuthology.task.kernel:src is distro, skipping download 2023-11-17T05:01:07.610 INFO:teuthology.task.kernel:Installing distro kernel on host.b... 2023-11-17T05:01:07.610 DEBUG:teuthology.task.kernel:install_kernel(remote=ubuntu@smithi066.front.sepia.ceph.com, path=None, version=distro) 2023-11-17T05:01:07.610 DEBUG:teuthology.orchestra.run.smithi066:> rpm -q kernel | sort -rV | head -n 1 2023-11-17T05:01:07.718 INFO:teuthology.orchestra.run.smithi040.stdout: Running scriptlet: kernel-4.18.0-521.el8.x86_64 3/3 2023-11-17T05:01:07.719 INFO:teuthology.orchestra.run.smithi040.stdout: Verifying : kernel-4.18.0-521.el8.x86_64 1/3 2023-11-17T05:01:07.719 INFO:teuthology.orchestra.run.smithi040.stdout: Verifying : kernel-core-4.18.0-521.el8.x86_64 2/3 2023-11-17T05:01:07.828 INFO:teuthology.orchestra.run.smithi066.stdout:kernel-4.18.0-521.el8.x86_64 2023-11-17T05:01:07.829 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 4.18.0-521.el8.x86_64 2023-11-17T05:01:07.829 DEBUG:teuthology.orchestra.run.smithi066:> sudo rpm -qi grub2-tools 2023-11-17T05:01:07.866 INFO:teuthology.orchestra.run.smithi066.stdout:Name : grub2-tools 2023-11-17T05:01:07.866 INFO:teuthology.orchestra.run.smithi066.stdout:Epoch : 1 2023-11-17T05:01:07.866 INFO:teuthology.orchestra.run.smithi066.stdout:Version : 2.02 2023-11-17T05:01:07.866 INFO:teuthology.orchestra.run.smithi066.stdout:Release : 129.el8 2023-11-17T05:01:07.867 INFO:teuthology.orchestra.run.smithi066.stdout:Architecture: x86_64 2023-11-17T05:01:07.867 INFO:teuthology.orchestra.run.smithi066.stdout:Install Date: Thu 04 May 2023 10:17:48 PM UTC 2023-11-17T05:01:07.867 INFO:teuthology.orchestra.run.smithi066.stdout:Group : System Environment/Base 2023-11-17T05:01:07.868 INFO:teuthology.orchestra.run.smithi066.stdout:Size : 9265586 2023-11-17T05:01:07.868 INFO:teuthology.orchestra.run.smithi066.stdout:License : GPLv3+ 2023-11-17T05:01:07.868 INFO:teuthology.orchestra.run.smithi066.stdout:Signature : RSA/SHA256, Tue 19 Jul 2022 02:52:06 PM UTC, Key ID 05b555b38483c65d 2023-11-17T05:01:07.868 INFO:teuthology.orchestra.run.smithi066.stdout:Source RPM : grub2-2.02-129.el8.src.rpm 2023-11-17T05:01:07.868 INFO:teuthology.orchestra.run.smithi066.stdout:Build Date : Tue 19 Jul 2022 02:30:39 PM UTC 2023-11-17T05:01:07.868 INFO:teuthology.orchestra.run.smithi066.stdout:Build Host : kbuilder.bsys.centos.org 2023-11-17T05:01:07.868 INFO:teuthology.orchestra.run.smithi066.stdout:Relocations : (not relocatable) 2023-11-17T05:01:07.868 INFO:teuthology.orchestra.run.smithi066.stdout:Packager : CentOS BuildSystem 2023-11-17T05:01:07.868 INFO:teuthology.orchestra.run.smithi066.stdout:Vendor : CentOS 2023-11-17T05:01:07.868 INFO:teuthology.orchestra.run.smithi066.stdout:URL : http://www.gnu.org/software/grub/ 2023-11-17T05:01:07.868 INFO:teuthology.orchestra.run.smithi066.stdout:Summary : Support tools for GRUB. 2023-11-17T05:01:07.868 INFO:teuthology.orchestra.run.smithi066.stdout:Description : 2023-11-17T05:01:07.868 INFO:teuthology.orchestra.run.smithi066.stdout: 2023-11-17T05:01:07.869 INFO:teuthology.orchestra.run.smithi066.stdout:The GRand Unified Bootloader (GRUB) is a highly configurable and 2023-11-17T05:01:07.869 INFO:teuthology.orchestra.run.smithi066.stdout:customizable bootloader with modular architecture. It supports a rich 2023-11-17T05:01:07.869 INFO:teuthology.orchestra.run.smithi066.stdout:variety of kernel formats, file systems, computer architectures and 2023-11-17T05:01:07.869 INFO:teuthology.orchestra.run.smithi066.stdout:hardware devices. 2023-11-17T05:01:07.869 INFO:teuthology.orchestra.run.smithi066.stdout: 2023-11-17T05:01:07.869 INFO:teuthology.orchestra.run.smithi066.stdout:This subpackage provides tools for support of all platforms. 2023-11-17T05:01:07.869 INFO:teuthology.task.kernel:Updating Grub Version: grub2 2023-11-17T05:01:07.869 INFO:teuthology.task.kernel:Updating grub on smithi066 to boot 4.18.0-521.el8.x86_64 2023-11-17T05:01:07.869 DEBUG:teuthology.orchestra.run.smithi066:> sudo grub2-mkconfig -o /boot/grub2/grub.cfg 2023-11-17T05:01:08.476 INFO:teuthology.orchestra.run.smithi040.stdout: Verifying : kernel-modules-4.18.0-521.el8.x86_64 3/3 2023-11-17T05:01:08.476 INFO:teuthology.orchestra.run.smithi040.stdout: 2023-11-17T05:01:08.476 INFO:teuthology.orchestra.run.smithi040.stdout:Installed: 2023-11-17T05:01:08.476 INFO:teuthology.orchestra.run.smithi040.stdout: kernel-4.18.0-521.el8.x86_64 kernel-core-4.18.0-521.el8.x86_64 2023-11-17T05:01:08.476 INFO:teuthology.orchestra.run.smithi040.stdout: kernel-modules-4.18.0-521.el8.x86_64 2023-11-17T05:01:08.476 INFO:teuthology.orchestra.run.smithi040.stdout: 2023-11-17T05:01:08.476 INFO:teuthology.orchestra.run.smithi040.stdout:Complete! 2023-11-17T05:01:08.628 DEBUG:teuthology.orchestra.run.smithi040:> rpm -q kernel | sort -rV | head -n 1 2023-11-17T05:01:08.877 INFO:teuthology.orchestra.run.smithi040.stdout:kernel-4.18.0-521.el8.x86_64 2023-11-17T05:01:08.877 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 4.18.0-521.el8.x86_64 2023-11-17T05:01:08.878 INFO:teuthology.task.kernel:Not newest distro kernel. Current: 4.18.0-489.el8.x86_64 Expected: 4.18.0-521.el8.x86_64 2023-11-17T05:01:08.878 INFO:teuthology.task.kernel:Skipping firmware on distro kernel 2023-11-17T05:01:08.878 DEBUG:teuthology.task.kernel:src is distro, skipping download 2023-11-17T05:01:08.878 INFO:teuthology.task.kernel:Installing distro kernel on host.a... 2023-11-17T05:01:08.878 DEBUG:teuthology.task.kernel:install_kernel(remote=ubuntu@smithi040.front.sepia.ceph.com, path=None, version=distro) 2023-11-17T05:01:08.878 DEBUG:teuthology.orchestra.run.smithi040:> rpm -q kernel | sort -rV | head -n 1 2023-11-17T05:01:09.097 INFO:teuthology.orchestra.run.smithi040.stdout:kernel-4.18.0-521.el8.x86_64 2023-11-17T05:01:09.097 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 4.18.0-521.el8.x86_64 2023-11-17T05:01:09.097 DEBUG:teuthology.orchestra.run.smithi040:> sudo rpm -qi grub2-tools 2023-11-17T05:01:09.131 INFO:teuthology.orchestra.run.smithi040.stdout:Name : grub2-tools 2023-11-17T05:01:09.132 INFO:teuthology.orchestra.run.smithi040.stdout:Epoch : 1 2023-11-17T05:01:09.132 INFO:teuthology.orchestra.run.smithi040.stdout:Version : 2.02 2023-11-17T05:01:09.132 INFO:teuthology.orchestra.run.smithi040.stdout:Release : 129.el8 2023-11-17T05:01:09.132 INFO:teuthology.orchestra.run.smithi040.stdout:Architecture: x86_64 2023-11-17T05:01:09.132 INFO:teuthology.orchestra.run.smithi040.stdout:Install Date: Thu 04 May 2023 10:17:48 PM UTC 2023-11-17T05:01:09.132 INFO:teuthology.orchestra.run.smithi040.stdout:Group : System Environment/Base 2023-11-17T05:01:09.132 INFO:teuthology.orchestra.run.smithi040.stdout:Size : 9265586 2023-11-17T05:01:09.132 INFO:teuthology.orchestra.run.smithi040.stdout:License : GPLv3+ 2023-11-17T05:01:09.132 INFO:teuthology.orchestra.run.smithi040.stdout:Signature : RSA/SHA256, Tue 19 Jul 2022 02:52:06 PM UTC, Key ID 05b555b38483c65d 2023-11-17T05:01:09.132 INFO:teuthology.orchestra.run.smithi040.stdout:Source RPM : grub2-2.02-129.el8.src.rpm 2023-11-17T05:01:09.132 INFO:teuthology.orchestra.run.smithi040.stdout:Build Date : Tue 19 Jul 2022 02:30:39 PM UTC 2023-11-17T05:01:09.132 INFO:teuthology.orchestra.run.smithi040.stdout:Build Host : kbuilder.bsys.centos.org 2023-11-17T05:01:09.133 INFO:teuthology.orchestra.run.smithi040.stdout:Relocations : (not relocatable) 2023-11-17T05:01:09.133 INFO:teuthology.orchestra.run.smithi040.stdout:Packager : CentOS BuildSystem 2023-11-17T05:01:09.133 INFO:teuthology.orchestra.run.smithi040.stdout:Vendor : CentOS 2023-11-17T05:01:09.133 INFO:teuthology.orchestra.run.smithi040.stdout:URL : http://www.gnu.org/software/grub/ 2023-11-17T05:01:09.133 INFO:teuthology.orchestra.run.smithi040.stdout:Summary : Support tools for GRUB. 2023-11-17T05:01:09.133 INFO:teuthology.orchestra.run.smithi040.stdout:Description : 2023-11-17T05:01:09.133 INFO:teuthology.orchestra.run.smithi040.stdout: 2023-11-17T05:01:09.133 INFO:teuthology.orchestra.run.smithi040.stdout:The GRand Unified Bootloader (GRUB) is a highly configurable and 2023-11-17T05:01:09.133 INFO:teuthology.orchestra.run.smithi040.stdout:customizable bootloader with modular architecture. It supports a rich 2023-11-17T05:01:09.133 INFO:teuthology.orchestra.run.smithi040.stdout:variety of kernel formats, file systems, computer architectures and 2023-11-17T05:01:09.133 INFO:teuthology.orchestra.run.smithi040.stdout:hardware devices. 2023-11-17T05:01:09.133 INFO:teuthology.orchestra.run.smithi040.stdout: 2023-11-17T05:01:09.133 INFO:teuthology.orchestra.run.smithi040.stdout:This subpackage provides tools for support of all platforms. 2023-11-17T05:01:09.134 INFO:teuthology.task.kernel:Updating Grub Version: grub2 2023-11-17T05:01:09.134 INFO:teuthology.task.kernel:Updating grub on smithi040 to boot 4.18.0-521.el8.x86_64 2023-11-17T05:01:09.135 DEBUG:teuthology.orchestra.run.smithi040:> sudo grub2-mkconfig -o /boot/grub2/grub.cfg 2023-11-17T05:01:09.188 INFO:teuthology.orchestra.run.smithi066.stderr:Generating grub configuration file ... 2023-11-17T05:01:10.732 INFO:teuthology.orchestra.run.smithi040.stderr:Generating grub configuration file ... 2023-11-17T05:01:10.931 INFO:teuthology.orchestra.run.smithi066.stderr:done 2023-11-17T05:01:10.933 DEBUG:teuthology.orchestra.run.smithi066:> mktemp 2023-11-17T05:01:10.958 INFO:teuthology.orchestra.run.smithi066.stdout:/tmp/tmp.dA4oyIOIl6 2023-11-17T05:01:10.958 DEBUG:teuthology.orchestra.run.smithi066:> sudo cp /boot/grub2/grub.cfg /tmp/tmp.dA4oyIOIl6 2023-11-17T05:01:11.029 DEBUG:teuthology.orchestra.run.smithi066:> sudo chmod 0666 /tmp/tmp.dA4oyIOIl6 2023-11-17T05:01:11.177 DEBUG:teuthology.orchestra.remote:smithi066:/tmp/tmp.dA4oyIOIl6 is 6KB 2023-11-17T05:01:11.224 DEBUG:teuthology.orchestra.run.smithi066:> rm -fr /tmp/tmp.dA4oyIOIl6 2023-11-17T05:01:11.242 DEBUG:teuthology.orchestra.run.smithi066:> sudo /bin/ls /boot/loader/entries || true 2023-11-17T05:01:11.309 INFO:teuthology.orchestra.run.smithi066.stdout:a1b29d6c289747bdb22930598d39874a-0-rescue.conf 2023-11-17T05:01:11.309 INFO:teuthology.orchestra.run.smithi066.stdout:a1b29d6c289747bdb22930598d39874a-4.18.0-348.el8.x86_64.conf 2023-11-17T05:01:11.309 INFO:teuthology.orchestra.run.smithi066.stdout:a1b29d6c289747bdb22930598d39874a-4.18.0-489.el8.x86_64.conf 2023-11-17T05:01:11.309 INFO:teuthology.orchestra.run.smithi066.stdout:a1b29d6c289747bdb22930598d39874a-4.18.0-521.el8.x86_64.conf 2023-11-17T05:01:11.310 DEBUG:teuthology.orchestra.run.smithi066:> sudo grub2-set-default a1b29d6c289747bdb22930598d39874a-4.18.0-521.el8.x86_64 2023-11-17T05:01:11.485 DEBUG:teuthology.orchestra.run.smithi066:> sudo shutdown -r now 2023-11-17T05:01:12.532 INFO:teuthology.orchestra.run.smithi040.stderr:done 2023-11-17T05:01:12.534 DEBUG:teuthology.orchestra.run.smithi040:> mktemp 2023-11-17T05:01:12.561 INFO:teuthology.orchestra.run.smithi040.stdout:/tmp/tmp.cuWCOydeDR 2023-11-17T05:01:12.562 DEBUG:teuthology.orchestra.run.smithi040:> sudo cp /boot/grub2/grub.cfg /tmp/tmp.cuWCOydeDR 2023-11-17T05:01:12.637 DEBUG:teuthology.orchestra.run.smithi040:> sudo chmod 0666 /tmp/tmp.cuWCOydeDR 2023-11-17T05:01:12.779 DEBUG:teuthology.orchestra.remote:smithi040:/tmp/tmp.cuWCOydeDR is 6KB 2023-11-17T05:01:12.827 DEBUG:teuthology.orchestra.run.smithi040:> rm -fr /tmp/tmp.cuWCOydeDR 2023-11-17T05:01:12.846 DEBUG:teuthology.orchestra.run.smithi040:> sudo /bin/ls /boot/loader/entries || true 2023-11-17T05:01:12.911 INFO:teuthology.orchestra.run.smithi040.stdout:a1b29d6c289747bdb22930598d39874a-0-rescue.conf 2023-11-17T05:01:12.911 INFO:teuthology.orchestra.run.smithi040.stdout:a1b29d6c289747bdb22930598d39874a-4.18.0-348.el8.x86_64.conf 2023-11-17T05:01:12.911 INFO:teuthology.orchestra.run.smithi040.stdout:a1b29d6c289747bdb22930598d39874a-4.18.0-489.el8.x86_64.conf 2023-11-17T05:01:12.911 INFO:teuthology.orchestra.run.smithi040.stdout:a1b29d6c289747bdb22930598d39874a-4.18.0-521.el8.x86_64.conf 2023-11-17T05:01:12.912 DEBUG:teuthology.orchestra.run.smithi040:> sudo grub2-set-default a1b29d6c289747bdb22930598d39874a-4.18.0-521.el8.x86_64 2023-11-17T05:01:13.048 DEBUG:teuthology.orchestra.run.smithi040:> sudo shutdown -r now 2023-11-17T05:01:41.514 INFO:teuthology.task.kernel:Checking client host.b for new kernel version... 2023-11-17T05:01:41.515 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi066.front.sepia.ceph.com' 2023-11-17T05:01:41.515 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi066.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-11-17T05:01:43.053 INFO:teuthology.task.kernel:Checking client host.a for new kernel version... 2023-11-17T05:01:46.671 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi040.front.sepia.ceph.com' 2023-11-17T05:01:46.686 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi040.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-11-17T05:02:00.058 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.66 2023-11-17T05:02:05.178 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.40 2023-11-17T05:03:02.715 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi066.front.sepia.ceph.com' 2023-11-17T05:03:02.716 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi066.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-11-17T05:03:05.790 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.66 2023-11-17T05:03:11.721 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi040.front.sepia.ceph.com' 2023-11-17T05:03:11.721 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi040.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-11-17T05:03:12.037 DEBUG:teuthology.orchestra.run.smithi040:> true 2023-11-17T05:03:12.304 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi040.front.sepia.ceph.com' 2023-11-17T05:03:12.304 INFO:teuthology.task.kernel:Checking kernel version of host.a, want "4.18.0-521.el8.x86_64"... 2023-11-17T05:03:12.305 DEBUG:teuthology.orchestra.run.smithi040:> uname -r 2023-11-17T05:03:12.326 INFO:teuthology.orchestra.run.smithi040.stdout:4.18.0-521.el8.x86_64 2023-11-17T05:03:12.327 DEBUG:teuthology.task.kernel:current kernel version is 4.18.0-521.el8.x86_64 vs 4.18.0-521.el8.x86_64 2023-11-17T05:03:12.327 DEBUG:teuthology.task.kernel:utsrelease strings match, do not need to install 2023-11-17T05:03:12.327 DEBUG:teuthology.task.kernel:Distro of this test job: centos 2023-11-17T05:03:13.327 INFO:teuthology.task.kernel:Enabling kdb on host.a... 2023-11-17T05:03:13.328 DEBUG:teuthology.orchestra.run.smithi040:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2023-11-17T05:03:13.415 INFO:teuthology.orchestra.run.smithi040.stdout:ttyS1 2023-11-17T05:03:13.428 DEBUG:teuthology.parallel:result is None 2023-11-17T05:03:17.792 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi066.front.sepia.ceph.com' 2023-11-17T05:03:17.792 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi066.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-11-17T05:03:20.858 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.66 2023-11-17T05:03:35.860 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi066.front.sepia.ceph.com' 2023-11-17T05:03:55.565 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi066.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-11-17T05:03:55.895 DEBUG:teuthology.orchestra.run.smithi066:> true 2023-11-17T05:03:56.222 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi066.front.sepia.ceph.com' 2023-11-17T05:03:56.222 INFO:teuthology.task.kernel:Checking kernel version of host.b, want "4.18.0-521.el8.x86_64"... 2023-11-17T05:03:56.223 DEBUG:teuthology.orchestra.run.smithi066:> uname -r 2023-11-17T05:03:56.243 INFO:teuthology.orchestra.run.smithi066.stdout:4.18.0-521.el8.x86_64 2023-11-17T05:03:56.243 DEBUG:teuthology.task.kernel:current kernel version is 4.18.0-521.el8.x86_64 vs 4.18.0-521.el8.x86_64 2023-11-17T05:03:56.243 DEBUG:teuthology.task.kernel:utsrelease strings match, do not need to install 2023-11-17T05:03:56.243 DEBUG:teuthology.task.kernel:Distro of this test job: centos 2023-11-17T05:03:57.244 INFO:teuthology.task.kernel:Enabling kdb on host.b... 2023-11-17T05:03:57.244 DEBUG:teuthology.orchestra.run.smithi066:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2023-11-17T05:03:57.331 INFO:teuthology.orchestra.run.smithi066.stdout:ttyS1 2023-11-17T05:03:57.344 DEBUG:teuthology.parallel:result is None 2023-11-17T05:03:57.344 INFO:teuthology.run_tasks:Running task internal.base... 2023-11-17T05:03:57.351 INFO:teuthology.task.internal:Creating test directory... 2023-11-17T05:03:57.351 DEBUG:teuthology.orchestra.run.smithi040:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2023-11-17T05:03:57.354 DEBUG:teuthology.orchestra.run.smithi066:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2023-11-17T05:03:57.376 INFO:teuthology.run_tasks:Running task internal.archive_upload... 2023-11-17T05:03:57.388 INFO:teuthology.run_tasks:Running task internal.archive... 2023-11-17T05:03:57.446 INFO:teuthology.task.internal:Creating archive directory... 2023-11-17T05:03:57.447 DEBUG:teuthology.orchestra.run.smithi040:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2023-11-17T05:03:57.449 DEBUG:teuthology.orchestra.run.smithi066:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2023-11-17T05:03:57.494 INFO:teuthology.run_tasks:Running task internal.coredump... 2023-11-17T05:03:57.504 INFO:teuthology.task.internal:Enabling coredump saving... 2023-11-17T05:03:57.505 DEBUG:teuthology.orchestra.run.smithi040:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2023-11-17T05:03:57.532 DEBUG:teuthology.orchestra.run.smithi066:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2023-11-17T05:03:57.567 INFO:teuthology.orchestra.run.smithi040.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2023-11-17T05:03:57.575 INFO:teuthology.orchestra.run.smithi066.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2023-11-17T05:03:57.579 INFO:teuthology.orchestra.run.smithi040.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2023-11-17T05:03:57.587 INFO:teuthology.orchestra.run.smithi066.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2023-11-17T05:03:57.981 INFO:teuthology.run_tasks:Running task internal.sudo... 2023-11-17T05:03:57.991 INFO:teuthology.task.internal:Configuring sudo... 2023-11-17T05:03:58.173 DEBUG:teuthology.orchestra.run.smithi040:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2023-11-17T05:03:58.178 DEBUG:teuthology.orchestra.run.smithi066:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2023-11-17T05:03:58.320 INFO:teuthology.run_tasks:Running task internal.syslog... 2023-11-17T05:03:58.329 INFO:teuthology.task.internal.syslog:Starting syslog monitoring... 2023-11-17T05:03:58.329 DEBUG:teuthology.orchestra.run.smithi040:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2023-11-17T05:03:58.332 DEBUG:teuthology.orchestra.run.smithi066:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2023-11-17T05:03:58.383 DEBUG:teuthology.orchestra.run.smithi040:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2023-11-17T05:03:58.437 DEBUG:teuthology.orchestra.run.smithi040:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2023-11-17T05:03:58.509 DEBUG:teuthology.orchestra.run.smithi040:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2023-11-17T05:03:58.552 DEBUG:teuthology.orchestra.run.smithi040:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2023-11-17T05:03:58.607 DEBUG:teuthology.orchestra.run.smithi040:> set -ex 2023-11-17T05:03:58.607 DEBUG:teuthology.orchestra.run.smithi040:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2023-11-17T05:03:58.681 DEBUG:teuthology.orchestra.run.smithi066:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2023-11-17T05:03:58.729 DEBUG:teuthology.orchestra.run.smithi066:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2023-11-17T05:03:58.791 DEBUG:teuthology.orchestra.run.smithi066:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2023-11-17T05:03:58.875 DEBUG:teuthology.orchestra.run.smithi066:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2023-11-17T05:03:58.926 DEBUG:teuthology.orchestra.run.smithi066:> set -ex 2023-11-17T05:03:58.926 DEBUG:teuthology.orchestra.run.smithi066:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2023-11-17T05:03:59.000 DEBUG:teuthology.orchestra.run.smithi040:> sudo service rsyslog restart 2023-11-17T05:03:59.004 DEBUG:teuthology.orchestra.run.smithi066:> sudo service rsyslog restart 2023-11-17T05:03:59.079 INFO:teuthology.orchestra.run.smithi040.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2023-11-17T05:03:59.102 INFO:teuthology.orchestra.run.smithi066.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2023-11-17T05:03:59.564 INFO:teuthology.run_tasks:Running task internal.timer... 2023-11-17T05:03:59.573 INFO:teuthology.task.internal:Starting timer... 2023-11-17T05:03:59.573 INFO:teuthology.run_tasks:Running task pcp... 2023-11-17T05:03:59.648 INFO:teuthology.run_tasks:Running task selinux... 2023-11-17T05:03:59.696 DEBUG:teuthology.task:Applying overrides for task selinux: {'whitelist': ['scontext=system_u:system_r:logrotate_t:s0']} 2023-11-17T05:03:59.696 DEBUG:teuthology.orchestra.run.smithi040:> sudo service auditd rotate 2023-11-17T05:03:59.790 INFO:teuthology.orchestra.run.smithi040.stdout:Rotating logs: 2023-11-17T05:03:59.793 DEBUG:teuthology.orchestra.run.smithi066:> sudo service auditd rotate 2023-11-17T05:03:59.877 INFO:teuthology.orchestra.run.smithi066.stdout:Rotating logs: 2023-11-17T05:03:59.877 DEBUG:teuthology.task.selinux:Getting current SELinux state 2023-11-17T05:03:59.878 DEBUG:teuthology.orchestra.run.smithi040:> /usr/sbin/getenforce 2023-11-17T05:03:59.950 INFO:teuthology.orchestra.run.smithi040.stdout:Permissive 2023-11-17T05:03:59.951 DEBUG:teuthology.orchestra.run.smithi066:> /usr/sbin/getenforce 2023-11-17T05:03:59.974 INFO:teuthology.orchestra.run.smithi066.stdout:Permissive 2023-11-17T05:03:59.974 DEBUG:teuthology.task.selinux:Existing SELinux modes: {'ubuntu@smithi040.front.sepia.ceph.com': 'permissive', 'ubuntu@smithi066.front.sepia.ceph.com': 'permissive'} 2023-11-17T05:03:59.975 DEBUG:teuthology.orchestra.run.smithi040:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 2023-11-17T05:04:00.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-11-17T05:04:00.015 DEBUG:teuthology.orchestra.run.smithi066:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 2023-11-17T05:04:00.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-11-17T05:04:00.049 INFO:teuthology.task.selinux:Putting SELinux into permissive mode 2023-11-17T05:04:00.050 DEBUG:teuthology.orchestra.run.smithi040:> sudo /usr/sbin/setenforce permissive 2023-11-17T05:04:00.087 DEBUG:teuthology.orchestra.run.smithi066:> sudo /usr/sbin/setenforce permissive 2023-11-17T05:04:00.118 INFO:teuthology.run_tasks:Running task ansible.cephlab... 2023-11-17T05:04:00.128 INFO:teuthology.repo_utils:/home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main was just updated or references a specific commit; assuming it is current 2023-11-17T05:04:00.128 INFO:teuthology.repo_utils:Resetting repo at /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main to origin/main 2023-11-17T05:04:00.148 INFO:teuthology.task.ansible:Playbook: [{'import_playbook': 'ansible_managed.yml'}, {'import_playbook': 'teuthology.yml'}, {'hosts': 'testnodes', 'tasks': [{'set_fact': {'ran_from_cephlab_playbook': True}}]}, {'import_playbook': 'testnodes.yml'}, {'import_playbook': 'container-host.yml'}, {'import_playbook': 'cobbler.yml'}, {'import_playbook': 'paddles.yml'}, {'import_playbook': 'pulpito.yml'}, {'hosts': 'testnodes', 'become': True, 'tasks': [{'name': 'Touch /ceph-qa-ready', 'file': {'path': '/ceph-qa-ready', 'state': 'touch'}, 'when': 'ran_from_cephlab_playbook|bool'}]}] 2023-11-17T05:04:00.149 DEBUG:teuthology.task.ansible:Running ansible-playbook -v --extra-vars '{"ansible_ssh_user": "ubuntu"}' -i /etc/ansible/hosts --limit smithi040.front.sepia.ceph.com,smithi066.front.sepia.ceph.com /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main/cephlab.yml 2023-11-17T05:13:20.004 DEBUG:teuthology.task.ansible:Reconnecting to [Remote(name='ubuntu@smithi040.front.sepia.ceph.com'), Remote(name='ubuntu@smithi066.front.sepia.ceph.com')] 2023-11-17T05:13:20.005 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi040.front.sepia.ceph.com' 2023-11-17T05:13:20.006 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi040.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-11-17T05:13:20.082 DEBUG:teuthology.orchestra.run.smithi040:> true 2023-11-17T05:13:20.161 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi040.front.sepia.ceph.com' 2023-11-17T05:13:20.161 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi066.front.sepia.ceph.com' 2023-11-17T05:13:20.162 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi066.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-11-17T05:13:20.232 DEBUG:teuthology.orchestra.run.smithi066:> true 2023-11-17T05:13:20.302 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi066.front.sepia.ceph.com' 2023-11-17T05:13:20.303 INFO:teuthology.run_tasks:Running task clock... 2023-11-17T05:13:20.314 INFO:teuthology.task.clock:Syncing clocks and checking initial clock skew... 2023-11-17T05:13:20.314 INFO:teuthology.orchestra.run:Running command with timeout 360 2023-11-17T05:13:20.315 DEBUG:teuthology.orchestra.run.smithi040:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2023-11-17T05:13:20.317 INFO:teuthology.orchestra.run:Running command with timeout 360 2023-11-17T05:13:20.317 DEBUG:teuthology.orchestra.run.smithi066:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2023-11-17T05:13:20.354 INFO:teuthology.orchestra.run.smithi040.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2023-11-17T05:13:20.371 INFO:teuthology.orchestra.run.smithi040.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2023-11-17T05:13:20.378 INFO:teuthology.orchestra.run.smithi066.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2023-11-17T05:13:20.392 INFO:teuthology.orchestra.run.smithi066.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2023-11-17T05:13:20.436 INFO:teuthology.orchestra.run.smithi040.stderr:sudo: ntpd: command not found 2023-11-17T05:13:20.449 INFO:teuthology.orchestra.run.smithi040.stdout:506 Cannot talk to daemon 2023-11-17T05:13:20.466 INFO:teuthology.orchestra.run.smithi040.stderr:Failed to start ntp.service: Unit ntp.service not found. 2023-11-17T05:13:20.472 INFO:teuthology.orchestra.run.smithi066.stderr:sudo: ntpd: command not found 2023-11-17T05:13:20.481 INFO:teuthology.orchestra.run.smithi040.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2023-11-17T05:13:20.484 INFO:teuthology.orchestra.run.smithi066.stdout:506 Cannot talk to daemon 2023-11-17T05:13:20.498 INFO:teuthology.orchestra.run.smithi066.stderr:Failed to start ntp.service: Unit ntp.service not found. 2023-11-17T05:13:20.512 INFO:teuthology.orchestra.run.smithi066.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2023-11-17T05:13:20.559 INFO:teuthology.orchestra.run.smithi040.stderr:bash: ntpq: command not found 2023-11-17T05:13:20.564 INFO:teuthology.orchestra.run.smithi040.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2023-11-17T05:13:20.564 INFO:teuthology.orchestra.run.smithi040.stdout:=============================================================================== 2023-11-17T05:13:20.564 INFO:teuthology.orchestra.run.smithi040.stdout:^? hv01.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2023-11-17T05:13:20.564 INFO:teuthology.orchestra.run.smithi040.stdout:^? hv02.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2023-11-17T05:13:20.564 INFO:teuthology.orchestra.run.smithi040.stdout:^? hv03.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2023-11-17T05:13:20.565 INFO:teuthology.orchestra.run.smithi040.stdout:^? hv04.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2023-11-17T05:13:20.577 INFO:teuthology.orchestra.run.smithi066.stderr:bash: ntpq: command not found 2023-11-17T05:13:20.581 INFO:teuthology.orchestra.run.smithi066.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2023-11-17T05:13:20.581 INFO:teuthology.orchestra.run.smithi066.stdout:=============================================================================== 2023-11-17T05:13:20.581 INFO:teuthology.orchestra.run.smithi066.stdout:^? hv01.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2023-11-17T05:13:20.581 INFO:teuthology.orchestra.run.smithi066.stdout:^? hv02.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2023-11-17T05:13:20.581 INFO:teuthology.orchestra.run.smithi066.stdout:^? hv03.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2023-11-17T05:13:20.581 INFO:teuthology.orchestra.run.smithi066.stdout:^? hv04.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2023-11-17T05:13:20.583 INFO:teuthology.run_tasks:Running task pexec... 2023-11-17T05:13:20.592 INFO:teuthology.task.pexec:Executing custom commands... 2023-11-17T05:13:20.593 INFO:teuthology.task.pexec:Running commands on host ubuntu@smithi040.front.sepia.ceph.com 2023-11-17T05:13:20.593 DEBUG:teuthology.orchestra.run.smithi040:> TESTDIR=/home/ubuntu/cephtest bash -s 2023-11-17T05:13:20.593 INFO:teuthology.task.pexec:Running commands on host ubuntu@smithi066.front.sepia.ceph.com 2023-11-17T05:13:20.593 DEBUG:teuthology.orchestra.run.smithi066:> TESTDIR=/home/ubuntu/cephtest bash -s 2023-11-17T05:13:21.535 INFO:teuthology.orchestra.run.smithi066.stdout:Last metadata expiration check: 0:02:38 ago on Fri 17 Nov 2023 05:10:43 AM UTC. 2023-11-17T05:13:21.618 INFO:teuthology.orchestra.run.smithi040.stdout:Last metadata expiration check: 0:02:06 ago on Fri 17 Nov 2023 05:11:15 AM UTC. 2023-11-17T05:13:23.809 INFO:teuthology.orchestra.run.smithi066.stdout:Dependencies resolved. 2023-11-17T05:13:23.810 INFO:teuthology.orchestra.run.smithi066.stdout:================================================================================ 2023-11-17T05:13:23.810 INFO:teuthology.orchestra.run.smithi066.stdout: Package Architecture Version Repository Size 2023-11-17T05:13:23.810 INFO:teuthology.orchestra.run.smithi066.stdout:================================================================================ 2023-11-17T05:13:23.810 INFO:teuthology.orchestra.run.smithi066.stdout:Resetting modules: 2023-11-17T05:13:23.810 INFO:teuthology.orchestra.run.smithi066.stdout: container-tools 2023-11-17T05:13:23.810 INFO:teuthology.orchestra.run.smithi066.stdout: 2023-11-17T05:13:23.810 INFO:teuthology.orchestra.run.smithi066.stdout:Transaction Summary 2023-11-17T05:13:23.810 INFO:teuthology.orchestra.run.smithi066.stdout:================================================================================ 2023-11-17T05:13:23.811 INFO:teuthology.orchestra.run.smithi066.stdout: 2023-11-17T05:13:23.979 INFO:teuthology.orchestra.run.smithi040.stdout:Dependencies resolved. 2023-11-17T05:13:23.980 INFO:teuthology.orchestra.run.smithi040.stdout:================================================================================ 2023-11-17T05:13:23.980 INFO:teuthology.orchestra.run.smithi040.stdout: Package Architecture Version Repository Size 2023-11-17T05:13:23.980 INFO:teuthology.orchestra.run.smithi040.stdout:================================================================================ 2023-11-17T05:13:23.980 INFO:teuthology.orchestra.run.smithi040.stdout:Resetting modules: 2023-11-17T05:13:23.980 INFO:teuthology.orchestra.run.smithi040.stdout: container-tools 2023-11-17T05:13:23.981 INFO:teuthology.orchestra.run.smithi040.stdout: 2023-11-17T05:13:23.981 INFO:teuthology.orchestra.run.smithi040.stdout:Transaction Summary 2023-11-17T05:13:23.981 INFO:teuthology.orchestra.run.smithi040.stdout:================================================================================ 2023-11-17T05:13:23.981 INFO:teuthology.orchestra.run.smithi040.stdout: 2023-11-17T05:13:24.073 INFO:teuthology.orchestra.run.smithi066.stdout:Complete! 2023-11-17T05:13:24.258 INFO:teuthology.orchestra.run.smithi040.stdout:Complete! 2023-11-17T05:13:25.073 INFO:teuthology.orchestra.run.smithi066.stdout:Last metadata expiration check: 0:02:41 ago on Fri 17 Nov 2023 05:10:43 AM UTC. 2023-11-17T05:13:25.362 INFO:teuthology.orchestra.run.smithi040.stdout:Last metadata expiration check: 0:02:10 ago on Fri 17 Nov 2023 05:11:15 AM UTC. 2023-11-17T05:13:27.326 INFO:teuthology.orchestra.run.smithi066.stdout:Dependencies resolved. 2023-11-17T05:13:27.519 INFO:teuthology.orchestra.run.smithi066.stdout:================================================================================ 2023-11-17T05:13:27.519 INFO:teuthology.orchestra.run.smithi066.stdout: Package Arch Version Repository Size 2023-11-17T05:13:27.519 INFO:teuthology.orchestra.run.smithi066.stdout:================================================================================ 2023-11-17T05:13:27.519 INFO:teuthology.orchestra.run.smithi066.stdout:Installing group/module packages: 2023-11-17T05:13:27.519 INFO:teuthology.orchestra.run.smithi066.stdout: aardvark-dns x86_64 2:1.8.0-1.module_el8+712+4cd1bd69 appstream 1.0 M 2023-11-17T05:13:27.519 INFO:teuthology.orchestra.run.smithi066.stdout: buildah x86_64 1:1.32.0-1.module_el8+712+4cd1bd69 appstream 9.6 M 2023-11-17T05:13:27.519 INFO:teuthology.orchestra.run.smithi066.stdout: cockpit-podman noarch 78-1.module_el8+712+4cd1bd69 appstream 750 k 2023-11-17T05:13:27.519 INFO:teuthology.orchestra.run.smithi066.stdout: crun x86_64 1.9.2-1.module_el8+712+4cd1bd69 appstream 241 k 2023-11-17T05:13:27.519 INFO:teuthology.orchestra.run.smithi066.stdout: netavark x86_64 2:1.8.0-2.module_el8+712+4cd1bd69 appstream 3.9 M 2023-11-17T05:13:27.519 INFO:teuthology.orchestra.run.smithi066.stdout: python3-podman noarch 4.7.0-1.module_el8+712+4cd1bd69 appstream 178 k 2023-11-17T05:13:27.520 INFO:teuthology.orchestra.run.smithi066.stdout: skopeo x86_64 2:1.13.3-1.module_el8+712+4cd1bd69 appstream 8.1 M 2023-11-17T05:13:27.520 INFO:teuthology.orchestra.run.smithi066.stdout: toolbox x86_64 0.0.99.4-6.module_el8+712+4cd1bd69 appstream 2.5 M 2023-11-17T05:13:27.520 INFO:teuthology.orchestra.run.smithi066.stdout: udica noarch 0.2.6-20.module_el8+712+4cd1bd69 appstream 53 k 2023-11-17T05:13:27.520 INFO:teuthology.orchestra.run.smithi066.stdout:Installing dependencies: 2023-11-17T05:13:27.520 INFO:teuthology.orchestra.run.smithi066.stdout: python3-pytoml noarch 0.1.14-5.git7dea353.el8 appstream 25 k 2023-11-17T05:13:27.520 INFO:teuthology.orchestra.run.smithi066.stdout: python3-pyxdg noarch 0.25-16.el8 appstream 94 k 2023-11-17T05:13:27.520 INFO:teuthology.orchestra.run.smithi066.stdout: yajl x86_64 2.1.0-12.el8 appstream 41 k 2023-11-17T05:13:27.520 INFO:teuthology.orchestra.run.smithi066.stdout:Installing weak dependencies: 2023-11-17T05:13:27.520 INFO:teuthology.orchestra.run.smithi066.stdout: criu-libs x86_64 3.15-3.module_el8.7.0+1216+b022c01d appstream 38 k 2023-11-17T05:13:27.520 INFO:teuthology.orchestra.run.smithi066.stdout:Installing module profiles: 2023-11-17T05:13:27.521 INFO:teuthology.orchestra.run.smithi066.stdout: container-tools/common 2023-11-17T05:13:27.521 INFO:teuthology.orchestra.run.smithi066.stdout: 2023-11-17T05:13:27.521 INFO:teuthology.orchestra.run.smithi066.stdout:Enabling module streams: 2023-11-17T05:13:27.521 INFO:teuthology.orchestra.run.smithi066.stdout: container-tools rhel8 2023-11-17T05:13:27.521 INFO:teuthology.orchestra.run.smithi066.stdout: 2023-11-17T05:13:27.521 INFO:teuthology.orchestra.run.smithi066.stdout:Transaction Summary 2023-11-17T05:13:27.521 INFO:teuthology.orchestra.run.smithi066.stdout:================================================================================ 2023-11-17T05:13:27.521 INFO:teuthology.orchestra.run.smithi066.stdout:Install 13 Packages 2023-11-17T05:13:27.521 INFO:teuthology.orchestra.run.smithi066.stdout: 2023-11-17T05:13:27.522 INFO:teuthology.orchestra.run.smithi066.stdout:Total download size: 27 M 2023-11-17T05:13:27.522 INFO:teuthology.orchestra.run.smithi066.stdout:Installed size: 89 M 2023-11-17T05:13:27.522 INFO:teuthology.orchestra.run.smithi066.stdout:Downloading Packages: 2023-11-17T05:13:27.741 INFO:teuthology.orchestra.run.smithi040.stdout:Dependencies resolved. 2023-11-17T05:13:27.940 INFO:teuthology.orchestra.run.smithi040.stdout:================================================================================ 2023-11-17T05:13:27.941 INFO:teuthology.orchestra.run.smithi040.stdout: Package Arch Version Repository Size 2023-11-17T05:13:27.941 INFO:teuthology.orchestra.run.smithi040.stdout:================================================================================ 2023-11-17T05:13:27.941 INFO:teuthology.orchestra.run.smithi040.stdout:Installing group/module packages: 2023-11-17T05:13:27.941 INFO:teuthology.orchestra.run.smithi040.stdout: aardvark-dns x86_64 2:1.8.0-1.module_el8+712+4cd1bd69 appstream 1.0 M 2023-11-17T05:13:27.941 INFO:teuthology.orchestra.run.smithi040.stdout: buildah x86_64 1:1.32.0-1.module_el8+712+4cd1bd69 appstream 9.6 M 2023-11-17T05:13:27.941 INFO:teuthology.orchestra.run.smithi040.stdout: cockpit-podman noarch 78-1.module_el8+712+4cd1bd69 appstream 750 k 2023-11-17T05:13:27.941 INFO:teuthology.orchestra.run.smithi040.stdout: crun x86_64 1.9.2-1.module_el8+712+4cd1bd69 appstream 241 k 2023-11-17T05:13:27.941 INFO:teuthology.orchestra.run.smithi040.stdout: netavark x86_64 2:1.8.0-2.module_el8+712+4cd1bd69 appstream 3.9 M 2023-11-17T05:13:27.941 INFO:teuthology.orchestra.run.smithi040.stdout: python3-podman noarch 4.7.0-1.module_el8+712+4cd1bd69 appstream 178 k 2023-11-17T05:13:27.941 INFO:teuthology.orchestra.run.smithi040.stdout: skopeo x86_64 2:1.13.3-1.module_el8+712+4cd1bd69 appstream 8.1 M 2023-11-17T05:13:27.941 INFO:teuthology.orchestra.run.smithi040.stdout: toolbox x86_64 0.0.99.4-6.module_el8+712+4cd1bd69 appstream 2.5 M 2023-11-17T05:13:27.941 INFO:teuthology.orchestra.run.smithi040.stdout: udica noarch 0.2.6-20.module_el8+712+4cd1bd69 appstream 53 k 2023-11-17T05:13:27.941 INFO:teuthology.orchestra.run.smithi040.stdout:Installing dependencies: 2023-11-17T05:13:27.942 INFO:teuthology.orchestra.run.smithi040.stdout: python3-pytoml noarch 0.1.14-5.git7dea353.el8 appstream 25 k 2023-11-17T05:13:27.942 INFO:teuthology.orchestra.run.smithi040.stdout: python3-pyxdg noarch 0.25-16.el8 appstream 94 k 2023-11-17T05:13:27.942 INFO:teuthology.orchestra.run.smithi040.stdout: yajl x86_64 2.1.0-12.el8 appstream 41 k 2023-11-17T05:13:27.942 INFO:teuthology.orchestra.run.smithi040.stdout:Installing weak dependencies: 2023-11-17T05:13:27.942 INFO:teuthology.orchestra.run.smithi040.stdout: criu-libs x86_64 3.15-3.module_el8.7.0+1216+b022c01d appstream 38 k 2023-11-17T05:13:27.942 INFO:teuthology.orchestra.run.smithi040.stdout:Installing module profiles: 2023-11-17T05:13:27.942 INFO:teuthology.orchestra.run.smithi040.stdout: container-tools/common 2023-11-17T05:13:27.942 INFO:teuthology.orchestra.run.smithi040.stdout: 2023-11-17T05:13:27.942 INFO:teuthology.orchestra.run.smithi040.stdout:Enabling module streams: 2023-11-17T05:13:27.942 INFO:teuthology.orchestra.run.smithi040.stdout: container-tools rhel8 2023-11-17T05:13:27.942 INFO:teuthology.orchestra.run.smithi040.stdout: 2023-11-17T05:13:27.942 INFO:teuthology.orchestra.run.smithi040.stdout:Transaction Summary 2023-11-17T05:13:27.942 INFO:teuthology.orchestra.run.smithi040.stdout:================================================================================ 2023-11-17T05:13:27.943 INFO:teuthology.orchestra.run.smithi040.stdout:Install 13 Packages 2023-11-17T05:13:27.943 INFO:teuthology.orchestra.run.smithi040.stdout: 2023-11-17T05:13:27.943 INFO:teuthology.orchestra.run.smithi040.stdout:Total download size: 27 M 2023-11-17T05:13:27.943 INFO:teuthology.orchestra.run.smithi040.stdout:Installed size: 89 M 2023-11-17T05:13:27.943 INFO:teuthology.orchestra.run.smithi040.stdout:Downloading Packages: 2023-11-17T05:13:28.028 INFO:teuthology.orchestra.run.smithi066.stdout:(1/13): aardvark-dns-1.8.0-1.module_el8+712+4cd 2.1 MB/s | 1.0 MB 00:00 2023-11-17T05:13:28.103 INFO:teuthology.orchestra.run.smithi066.stdout:(2/13): criu-libs-3.15-3.module_el8.7.0+1216+b0 506 kB/s | 38 kB 00:00 2023-11-17T05:13:28.138 INFO:teuthology.orchestra.run.smithi066.stdout:(3/13): cockpit-podman-78-1.module_el8+712+4cd1 1.3 MB/s | 750 kB 00:00 2023-11-17T05:13:28.204 INFO:teuthology.orchestra.run.smithi066.stdout:(4/13): crun-1.9.2-1.module_el8+712+4cd1bd69.x8 2.3 MB/s | 241 kB 00:00 2023-11-17T05:13:28.396 INFO:teuthology.orchestra.run.smithi066.stdout:(5/13): python3-podman-4.7.0-1.module_el8+712+4 923 kB/s | 178 kB 00:00 2023-11-17T05:13:28.479 INFO:teuthology.orchestra.run.smithi066.stdout:(6/13): python3-pytoml-0.1.14-5.git7dea353.el8. 300 kB/s | 25 kB 00:00 2023-11-17T05:13:28.554 INFO:teuthology.orchestra.run.smithi066.stdout:(7/13): python3-pyxdg-0.25-16.el8.noarch.rpm 1.2 MB/s | 94 kB 00:00 2023-11-17T05:13:28.709 INFO:teuthology.orchestra.run.smithi040.stdout:(1/13): cockpit-podman-78-1.module_el8+712+4cd1 1.3 MB/s | 750 kB 00:00 2023-11-17T05:13:28.776 INFO:teuthology.orchestra.run.smithi040.stdout:(2/13): criu-libs-3.15-3.module_el8.7.0+1216+b0 571 kB/s | 38 kB 00:00 2023-11-17T05:13:28.868 INFO:teuthology.orchestra.run.smithi040.stdout:(3/13): aardvark-dns-1.8.0-1.module_el8+712+4cd 1.4 MB/s | 1.0 MB 00:00 2023-11-17T05:13:28.935 INFO:teuthology.orchestra.run.smithi040.stdout:(4/13): crun-1.9.2-1.module_el8+712+4cd1bd69.x8 1.5 MB/s | 241 kB 00:00 2023-11-17T05:13:29.035 INFO:teuthology.orchestra.run.smithi040.stdout:(5/13): python3-podman-4.7.0-1.module_el8+712+4 1.7 MB/s | 178 kB 00:00 2023-11-17T05:13:29.094 INFO:teuthology.orchestra.run.smithi040.stdout:(6/13): python3-pytoml-0.1.14-5.git7dea353.el8. 425 kB/s | 25 kB 00:00 2023-11-17T05:13:29.202 INFO:teuthology.orchestra.run.smithi040.stdout:(7/13): python3-pyxdg-0.25-16.el8.noarch.rpm 868 kB/s | 94 kB 00:00 2023-11-17T05:13:29.302 INFO:teuthology.orchestra.run.smithi066.stdout:(8/13): buildah-1.32.0-1.module_el8+712+4cd1bd6 5.5 MB/s | 9.6 MB 00:01 2023-11-17T05:13:29.363 INFO:teuthology.orchestra.run.smithi066.stdout:(9/13): netavark-1.8.0-2.module_el8+712+4cd1bd6 3.2 MB/s | 3.9 MB 00:01 2023-11-17T05:13:29.422 INFO:teuthology.orchestra.run.smithi066.stdout:(10/13): udica-0.2.6-20.module_el8+712+4cd1bd69 903 kB/s | 53 kB 00:00 2023-11-17T05:13:29.489 INFO:teuthology.orchestra.run.smithi066.stdout:(11/13): yajl-2.1.0-12.el8.x86_64.rpm 615 kB/s | 41 kB 00:00 2023-11-17T05:13:29.656 INFO:teuthology.orchestra.run.smithi066.stdout:(12/13): toolbox-0.0.99.4-6.module_el8+712+4cd1 7.2 MB/s | 2.5 MB 00:00 2023-11-17T05:13:29.870 INFO:teuthology.orchestra.run.smithi040.stdout:(8/13): netavark-1.8.0-2.module_el8+712+4cd1bd6 3.9 MB/s | 3.9 MB 00:01 2023-11-17T05:13:29.931 INFO:teuthology.orchestra.run.smithi066.stdout:(13/13): skopeo-1.13.3-1.module_el8+712+4cd1bd6 5.9 MB/s | 8.1 MB 00:01 2023-11-17T05:13:29.931 INFO:teuthology.orchestra.run.smithi066.stdout:-------------------------------------------------------------------------------- 2023-11-17T05:13:29.932 INFO:teuthology.orchestra.run.smithi066.stdout:Total 11 MB/s | 27 MB 00:02 2023-11-17T05:13:30.214 INFO:teuthology.orchestra.run.smithi066.stdout:Running transaction check 2023-11-17T05:13:30.236 INFO:teuthology.orchestra.run.smithi066.stdout:Transaction check succeeded. 2023-11-17T05:13:30.236 INFO:teuthology.orchestra.run.smithi066.stdout:Running transaction test 2023-11-17T05:13:30.311 INFO:teuthology.orchestra.run.smithi040.stdout:(9/13): buildah-1.32.0-1.module_el8+712+4cd1bd6 4.4 MB/s | 9.6 MB 00:02 2023-11-17T05:13:30.362 INFO:teuthology.orchestra.run.smithi040.stdout:(10/13): toolbox-0.0.99.4-6.module_el8+712+4cd1 5.1 MB/s | 2.5 MB 00:00 2023-11-17T05:13:30.378 INFO:teuthology.orchestra.run.smithi040.stdout:(11/13): udica-0.2.6-20.module_el8+712+4cd1bd69 791 kB/s | 53 kB 00:00 2023-11-17T05:13:30.413 INFO:teuthology.orchestra.run.smithi040.stdout:(12/13): yajl-2.1.0-12.el8.x86_64.rpm 816 kB/s | 41 kB 00:00 2023-11-17T05:13:30.495 INFO:teuthology.orchestra.run.smithi066.stdout:Transaction test succeeded. 2023-11-17T05:13:30.498 INFO:teuthology.orchestra.run.smithi066.stdout:Running transaction 2023-11-17T05:13:31.313 INFO:teuthology.orchestra.run.smithi040.stdout:(13/13): skopeo-1.13.3-1.module_el8+712+4cd1bd6 3.9 MB/s | 8.1 MB 00:02 2023-11-17T05:13:31.313 INFO:teuthology.orchestra.run.smithi040.stdout:-------------------------------------------------------------------------------- 2023-11-17T05:13:31.313 INFO:teuthology.orchestra.run.smithi040.stdout:Total 7.9 MB/s | 27 MB 00:03 2023-11-17T05:13:31.339 INFO:teuthology.orchestra.run.smithi066.stdout: Preparing : 1/1 2023-11-17T05:13:31.584 INFO:teuthology.orchestra.run.smithi066.stdout: Installing : yajl-2.1.0-12.el8.x86_64 1/13 2023-11-17T05:13:31.603 INFO:teuthology.orchestra.run.smithi040.stdout:Running transaction check 2023-11-17T05:13:31.628 INFO:teuthology.orchestra.run.smithi040.stdout:Transaction check succeeded. 2023-11-17T05:13:31.628 INFO:teuthology.orchestra.run.smithi040.stdout:Running transaction test 2023-11-17T05:13:31.761 INFO:teuthology.orchestra.run.smithi066.stdout: Installing : python3-pyxdg-0.25-16.el8.noarch 2/13 2023-11-17T05:13:31.915 INFO:teuthology.orchestra.run.smithi040.stdout:Transaction test succeeded. 2023-11-17T05:13:31.965 INFO:teuthology.orchestra.run.smithi066.stdout: Installing : python3-pytoml-0.1.14-5.git7dea353.el8.noarch 3/13 2023-11-17T05:13:31.965 INFO:teuthology.orchestra.run.smithi040.stdout:Running transaction 2023-11-17T05:13:32.160 INFO:teuthology.orchestra.run.smithi066.stdout: Installing : criu-libs-3.15-3.module_el8.7.0+1216+b022c01d.x86_ 4/13 2023-11-17T05:13:32.619 INFO:teuthology.orchestra.run.smithi040.stdout: Preparing : 1/1 2023-11-17T05:13:32.728 INFO:teuthology.orchestra.run.smithi066.stdout: Installing : aardvark-dns-2:1.8.0-1.module_el8+712+4cd1bd69.x86 5/13 2023-11-17T05:13:32.826 INFO:teuthology.orchestra.run.smithi040.stdout: Installing : yajl-2.1.0-12.el8.x86_64 1/13 2023-11-17T05:13:32.985 INFO:teuthology.orchestra.run.smithi040.stdout: Installing : python3-pyxdg-0.25-16.el8.noarch 2/13 2023-11-17T05:13:33.307 INFO:teuthology.orchestra.run.smithi040.stdout: Installing : python3-pytoml-0.1.14-5.git7dea353.el8.noarch 3/13 2023-11-17T05:13:33.538 INFO:teuthology.orchestra.run.smithi040.stdout: Installing : criu-libs-3.15-3.module_el8.7.0+1216+b022c01d.x86_ 4/13 2023-11-17T05:13:33.734 INFO:teuthology.orchestra.run.smithi066.stdout: Installing : netavark-2:1.8.0-2.module_el8+712+4cd1bd69.x86_64 6/13 2023-11-17T05:13:33.955 INFO:teuthology.orchestra.run.smithi066.stdout: Installing : buildah-1:1.32.0-1.module_el8+712+4cd1bd69.x86_64 7/13 2023-11-17T05:13:34.135 INFO:teuthology.orchestra.run.smithi040.stdout: Installing : aardvark-dns-2:1.8.0-1.module_el8+712+4cd1bd69.x86 5/13 2023-11-17T05:13:34.151 INFO:teuthology.orchestra.run.smithi066.stdout: Installing : crun-1.9.2-1.module_el8+712+4cd1bd69.x86_64 8/13 2023-11-17T05:13:34.424 INFO:teuthology.orchestra.run.smithi066.stdout: Installing : python3-podman-4.7.0-1.module_el8+712+4cd1bd69.noa 9/13 2023-11-17T05:13:34.946 INFO:teuthology.orchestra.run.smithi066.stdout: Installing : udica-0.2.6-20.module_el8+712+4cd1bd69.noarch 10/13 2023-11-17T05:13:35.211 INFO:teuthology.orchestra.run.smithi040.stdout: Installing : netavark-2:1.8.0-2.module_el8+712+4cd1bd69.x86_64 6/13 2023-11-17T05:13:35.454 INFO:teuthology.orchestra.run.smithi040.stdout: Installing : buildah-1:1.32.0-1.module_el8+712+4cd1bd69.x86_64 7/13 2023-11-17T05:13:35.680 INFO:teuthology.orchestra.run.smithi040.stdout: Installing : crun-1.9.2-1.module_el8+712+4cd1bd69.x86_64 8/13 2023-11-17T05:13:35.871 INFO:teuthology.orchestra.run.smithi066.stdout: Installing : toolbox-0.0.99.4-6.module_el8+712+4cd1bd69.x86_64 11/13 2023-11-17T05:13:35.907 INFO:teuthology.orchestra.run.smithi040.stdout: Installing : python3-podman-4.7.0-1.module_el8+712+4cd1bd69.noa 9/13 2023-11-17T05:13:36.073 INFO:teuthology.orchestra.run.smithi066.stdout: Installing : skopeo-2:1.13.3-1.module_el8+712+4cd1bd69.x86_64 12/13 2023-11-17T05:13:36.277 INFO:teuthology.orchestra.run.smithi066.stdout: Installing : cockpit-podman-78-1.module_el8+712+4cd1bd69.noarch 13/13 2023-11-17T05:13:36.306 INFO:teuthology.orchestra.run.smithi040.stdout: Installing : udica-0.2.6-20.module_el8+712+4cd1bd69.noarch 10/13 2023-11-17T05:13:36.926 INFO:teuthology.orchestra.run.smithi066.stdout: Running scriptlet: cockpit-podman-78-1.module_el8+712+4cd1bd69.noarch 13/13 2023-11-17T05:13:36.926 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : aardvark-dns-2:1.8.0-1.module_el8+712+4cd1bd69.x86 1/13 2023-11-17T05:13:36.927 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : buildah-1:1.32.0-1.module_el8+712+4cd1bd69.x86_64 2/13 2023-11-17T05:13:36.927 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : cockpit-podman-78-1.module_el8+712+4cd1bd69.noarch 3/13 2023-11-17T05:13:36.927 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : criu-libs-3.15-3.module_el8.7.0+1216+b022c01d.x86_ 4/13 2023-11-17T05:13:36.927 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : crun-1.9.2-1.module_el8+712+4cd1bd69.x86_64 5/13 2023-11-17T05:13:36.927 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : netavark-2:1.8.0-2.module_el8+712+4cd1bd69.x86_64 6/13 2023-11-17T05:13:36.927 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : python3-podman-4.7.0-1.module_el8+712+4cd1bd69.noa 7/13 2023-11-17T05:13:36.927 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : python3-pytoml-0.1.14-5.git7dea353.el8.noarch 8/13 2023-11-17T05:13:36.927 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : python3-pyxdg-0.25-16.el8.noarch 9/13 2023-11-17T05:13:36.927 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : skopeo-2:1.13.3-1.module_el8+712+4cd1bd69.x86_64 10/13 2023-11-17T05:13:36.927 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : toolbox-0.0.99.4-6.module_el8+712+4cd1bd69.x86_64 11/13 2023-11-17T05:13:36.927 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : udica-0.2.6-20.module_el8+712+4cd1bd69.noarch 12/13 2023-11-17T05:13:37.258 INFO:teuthology.orchestra.run.smithi040.stdout: Installing : toolbox-0.0.99.4-6.module_el8+712+4cd1bd69.x86_64 11/13 2023-11-17T05:13:42.506 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : yajl-2.1.0-12.el8.x86_64 13/13 2023-11-17T05:13:42.506 INFO:teuthology.orchestra.run.smithi066.stdout: 2023-11-17T05:13:42.506 INFO:teuthology.orchestra.run.smithi066.stdout:Installed: 2023-11-17T05:13:42.506 INFO:teuthology.orchestra.run.smithi066.stdout: aardvark-dns-2:1.8.0-1.module_el8+712+4cd1bd69.x86_64 2023-11-17T05:13:42.506 INFO:teuthology.orchestra.run.smithi066.stdout: buildah-1:1.32.0-1.module_el8+712+4cd1bd69.x86_64 2023-11-17T05:13:42.507 INFO:teuthology.orchestra.run.smithi066.stdout: cockpit-podman-78-1.module_el8+712+4cd1bd69.noarch 2023-11-17T05:13:42.507 INFO:teuthology.orchestra.run.smithi066.stdout: criu-libs-3.15-3.module_el8.7.0+1216+b022c01d.x86_64 2023-11-17T05:13:42.507 INFO:teuthology.orchestra.run.smithi066.stdout: crun-1.9.2-1.module_el8+712+4cd1bd69.x86_64 2023-11-17T05:13:42.507 INFO:teuthology.orchestra.run.smithi066.stdout: netavark-2:1.8.0-2.module_el8+712+4cd1bd69.x86_64 2023-11-17T05:13:42.507 INFO:teuthology.orchestra.run.smithi066.stdout: python3-podman-4.7.0-1.module_el8+712+4cd1bd69.noarch 2023-11-17T05:13:42.507 INFO:teuthology.orchestra.run.smithi066.stdout: python3-pytoml-0.1.14-5.git7dea353.el8.noarch 2023-11-17T05:13:42.507 INFO:teuthology.orchestra.run.smithi066.stdout: python3-pyxdg-0.25-16.el8.noarch 2023-11-17T05:13:42.507 INFO:teuthology.orchestra.run.smithi066.stdout: skopeo-2:1.13.3-1.module_el8+712+4cd1bd69.x86_64 2023-11-17T05:13:42.507 INFO:teuthology.orchestra.run.smithi066.stdout: toolbox-0.0.99.4-6.module_el8+712+4cd1bd69.x86_64 2023-11-17T05:13:42.507 INFO:teuthology.orchestra.run.smithi066.stdout: udica-0.2.6-20.module_el8+712+4cd1bd69.noarch 2023-11-17T05:13:42.507 INFO:teuthology.orchestra.run.smithi066.stdout: yajl-2.1.0-12.el8.x86_64 2023-11-17T05:13:42.507 INFO:teuthology.orchestra.run.smithi066.stdout: 2023-11-17T05:13:42.507 INFO:teuthology.orchestra.run.smithi066.stdout:Complete! 2023-11-17T05:13:42.508 DEBUG:teuthology.parallel:result is None 2023-11-17T05:13:42.509 INFO:teuthology.orchestra.run.smithi040.stdout: Installing : skopeo-2:1.13.3-1.module_el8+712+4cd1bd69.x86_64 12/13 2023-11-17T05:13:42.509 INFO:teuthology.orchestra.run.smithi040.stdout: Installing : cockpit-podman-78-1.module_el8+712+4cd1bd69.noarch 13/13 2023-11-17T05:13:42.509 INFO:teuthology.orchestra.run.smithi040.stdout: Running scriptlet: cockpit-podman-78-1.module_el8+712+4cd1bd69.noarch 13/13 2023-11-17T05:13:42.509 INFO:teuthology.orchestra.run.smithi040.stdout: Verifying : aardvark-dns-2:1.8.0-1.module_el8+712+4cd1bd69.x86 1/13 2023-11-17T05:13:42.510 INFO:teuthology.orchestra.run.smithi040.stdout: Verifying : buildah-1:1.32.0-1.module_el8+712+4cd1bd69.x86_64 2/13 2023-11-17T05:13:42.510 INFO:teuthology.orchestra.run.smithi040.stdout: Verifying : cockpit-podman-78-1.module_el8+712+4cd1bd69.noarch 3/13 2023-11-17T05:13:42.510 INFO:teuthology.orchestra.run.smithi040.stdout: Verifying : criu-libs-3.15-3.module_el8.7.0+1216+b022c01d.x86_ 4/13 2023-11-17T05:13:42.510 INFO:teuthology.orchestra.run.smithi040.stdout: Verifying : crun-1.9.2-1.module_el8+712+4cd1bd69.x86_64 5/13 2023-11-17T05:13:42.518 INFO:teuthology.orchestra.run.smithi040.stdout: Verifying : netavark-2:1.8.0-2.module_el8+712+4cd1bd69.x86_64 6/13 2023-11-17T05:13:42.640 INFO:teuthology.orchestra.run.smithi040.stdout: Verifying : python3-podman-4.7.0-1.module_el8+712+4cd1bd69.noa 7/13 2023-11-17T05:13:42.640 INFO:teuthology.orchestra.run.smithi040.stdout: Verifying : python3-pytoml-0.1.14-5.git7dea353.el8.noarch 8/13 2023-11-17T05:13:42.640 INFO:teuthology.orchestra.run.smithi040.stdout: Verifying : python3-pyxdg-0.25-16.el8.noarch 9/13 2023-11-17T05:13:42.640 INFO:teuthology.orchestra.run.smithi040.stdout: Verifying : skopeo-2:1.13.3-1.module_el8+712+4cd1bd69.x86_64 10/13 2023-11-17T05:13:42.640 INFO:teuthology.orchestra.run.smithi040.stdout: Verifying : toolbox-0.0.99.4-6.module_el8+712+4cd1bd69.x86_64 11/13 2023-11-17T05:13:42.641 INFO:teuthology.orchestra.run.smithi040.stdout: Verifying : udica-0.2.6-20.module_el8+712+4cd1bd69.noarch 12/13 2023-11-17T05:13:42.641 INFO:teuthology.orchestra.run.smithi040.stdout: Verifying : yajl-2.1.0-12.el8.x86_64 13/13 2023-11-17T05:13:42.641 INFO:teuthology.orchestra.run.smithi040.stdout: 2023-11-17T05:13:42.641 INFO:teuthology.orchestra.run.smithi040.stdout:Installed: 2023-11-17T05:13:42.641 INFO:teuthology.orchestra.run.smithi040.stdout: aardvark-dns-2:1.8.0-1.module_el8+712+4cd1bd69.x86_64 2023-11-17T05:13:42.641 INFO:teuthology.orchestra.run.smithi040.stdout: buildah-1:1.32.0-1.module_el8+712+4cd1bd69.x86_64 2023-11-17T05:13:42.641 INFO:teuthology.orchestra.run.smithi040.stdout: cockpit-podman-78-1.module_el8+712+4cd1bd69.noarch 2023-11-17T05:13:45.396 INFO:teuthology.orchestra.run.smithi040.stdout: criu-libs-3.15-3.module_el8.7.0+1216+b022c01d.x86_64 2023-11-17T05:13:45.396 INFO:teuthology.orchestra.run.smithi040.stdout: crun-1.9.2-1.module_el8+712+4cd1bd69.x86_64 2023-11-17T05:13:45.396 INFO:teuthology.orchestra.run.smithi040.stdout: netavark-2:1.8.0-2.module_el8+712+4cd1bd69.x86_64 2023-11-17T05:13:45.396 INFO:teuthology.orchestra.run.smithi040.stdout: python3-podman-4.7.0-1.module_el8+712+4cd1bd69.noarch 2023-11-17T05:13:45.396 INFO:teuthology.orchestra.run.smithi040.stdout: python3-pytoml-0.1.14-5.git7dea353.el8.noarch 2023-11-17T05:13:45.396 INFO:teuthology.orchestra.run.smithi040.stdout: python3-pyxdg-0.25-16.el8.noarch 2023-11-17T05:13:45.396 INFO:teuthology.orchestra.run.smithi040.stdout: skopeo-2:1.13.3-1.module_el8+712+4cd1bd69.x86_64 2023-11-17T05:13:45.396 INFO:teuthology.orchestra.run.smithi040.stdout: toolbox-0.0.99.4-6.module_el8+712+4cd1bd69.x86_64 2023-11-17T05:13:45.396 INFO:teuthology.orchestra.run.smithi040.stdout: udica-0.2.6-20.module_el8+712+4cd1bd69.noarch 2023-11-17T05:13:45.397 INFO:teuthology.orchestra.run.smithi040.stdout: yajl-2.1.0-12.el8.x86_64 2023-11-17T05:13:45.397 INFO:teuthology.orchestra.run.smithi040.stdout: 2023-11-17T05:13:45.397 INFO:teuthology.orchestra.run.smithi040.stdout:Complete! 2023-11-17T05:13:45.397 DEBUG:teuthology.parallel:result is None 2023-11-17T05:13:45.397 INFO:teuthology.run_tasks:Running task pexec... 2023-11-17T05:13:45.406 INFO:teuthology.task.pexec:Executing custom commands... 2023-11-17T05:13:45.406 INFO:teuthology.task.pexec:Running commands on host ubuntu@smithi040.front.sepia.ceph.com 2023-11-17T05:13:45.406 DEBUG:teuthology.orchestra.run.smithi040:> TESTDIR=/home/ubuntu/cephtest bash -s 2023-11-17T05:13:45.407 INFO:teuthology.task.pexec:Running commands on host ubuntu@smithi066.front.sepia.ceph.com 2023-11-17T05:13:45.407 DEBUG:teuthology.orchestra.run.smithi066:> TESTDIR=/home/ubuntu/cephtest bash -s 2023-11-17T05:13:45.568 DEBUG:teuthology.parallel:result is None 2023-11-17T05:13:45.572 DEBUG:teuthology.parallel:result is None 2023-11-17T05:13:45.572 INFO:teuthology.run_tasks:Running task install... 2023-11-17T05:13:45.581 DEBUG:teuthology.task.install:project ceph 2023-11-17T05:13:45.581 DEBUG:teuthology.task.install:INSTALL overrides: {'ceph': {'flavor': 'default', 'sha1': 'ed3c8e926b9de93026c8a7c1d86be5d6821b52ac'}} 2023-11-17T05:13:45.581 DEBUG:teuthology.task.install:config {'exclude_packages': ['ceph-volume'], 'tag': 'v16.2.4', 'flavor': 'default', 'sha1': 'ed3c8e926b9de93026c8a7c1d86be5d6821b52ac'} 2023-11-17T05:13:45.581 INFO:teuthology.task.install:Using flavor: default 2023-11-17T05:13:45.588 DEBUG:teuthology.task.install:Package list is: {'deb': ['ceph', 'cephadm', 'ceph-mds', 'ceph-mgr', 'ceph-common', 'ceph-fuse', 'ceph-test', 'radosgw', 'python3-rados', 'python3-rgw', 'python3-cephfs', 'python3-rbd', 'libcephfs2', 'libcephfs-dev', 'librados2', 'librbd1', 'rbd-fuse'], 'rpm': ['ceph-radosgw', 'ceph-test', 'ceph', 'ceph-base', 'cephadm', 'ceph-immutable-object-cache', 'ceph-mgr', 'ceph-mgr-dashboard', 'ceph-mgr-diskprediction-local', 'ceph-mgr-rook', 'ceph-mgr-cephadm', 'ceph-fuse', 'librados-devel', 'libcephfs2', 'libcephfs-devel', 'librados2', 'librbd1', 'python3-rados', 'python3-rgw', 'python3-cephfs', 'python3-rbd', 'rbd-fuse', 'rbd-mirror', 'rbd-nbd']} 2023-11-17T05:13:45.588 INFO:teuthology.task.install:extra packages: [] 2023-11-17T05:13:45.588 DEBUG:teuthology.task.install.rpm:_update_package_list_and_install: config is {'branch': None, 'cleanup': None, 'debuginfo': None, 'downgrade_packages': [], 'exclude_packages': ['ceph-volume'], 'extra_packages': [], 'extra_system_packages': [], 'extras': None, 'enable_coprs': [], 'flavor': 'default', 'install_ceph_packages': True, 'packages': {}, 'project': 'ceph', 'repos_only': False, 'sha1': 'ed3c8e926b9de93026c8a7c1d86be5d6821b52ac', 'tag': 'v16.2.4', 'wait_for_package': False} 2023-11-17T05:13:45.588 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using tag 2023-11-17T05:13:45.588 INFO:teuthology.packaging:ref: None 2023-11-17T05:13:45.588 INFO:teuthology.packaging:tag: v16.2.4 2023-11-17T05:13:45.588 INFO:teuthology.packaging:branch: None 2023-11-17T05:13:45.588 INFO:teuthology.packaging:sha1: ed3c8e926b9de93026c8a7c1d86be5d6821b52ac 2023-11-17T05:13:45.765 DEBUG:teuthology.repo_utils:git ls-remote https://github.com/ceph/ceph-ci v16.2.4^{} -> 3cbe25cde3cfa028984618ad32de9edc4c1eaed0 2023-11-17T05:14:40.843 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F8%2Fx86_64&sha1=3cbe25cde3cfa028984618ad32de9edc4c1eaed0 2023-11-17T05:14:40.993 DEBUG:teuthology.task.install.rpm:_update_package_list_and_install: config is {'branch': None, 'cleanup': None, 'debuginfo': None, 'downgrade_packages': [], 'exclude_packages': ['ceph-volume'], 'extra_packages': [], 'extra_system_packages': [], 'extras': None, 'enable_coprs': [], 'flavor': 'default', 'install_ceph_packages': True, 'packages': {}, 'project': 'ceph', 'repos_only': False, 'sha1': 'ed3c8e926b9de93026c8a7c1d86be5d6821b52ac', 'tag': 'v16.2.4', 'wait_for_package': False} 2023-11-17T05:14:40.993 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using tag 2023-11-17T05:14:40.993 INFO:teuthology.packaging:ref: None 2023-11-17T05:14:40.993 INFO:teuthology.packaging:tag: v16.2.4 2023-11-17T05:14:40.993 INFO:teuthology.packaging:branch: None 2023-11-17T05:14:40.994 INFO:teuthology.packaging:sha1: ed3c8e926b9de93026c8a7c1d86be5d6821b52ac 2023-11-17T05:14:41.208 DEBUG:teuthology.repo_utils:git ls-remote https://github.com/ceph/ceph-ci v16.2.4^{} -> 3cbe25cde3cfa028984618ad32de9edc4c1eaed0 2023-11-17T05:14:41.208 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F8%2Fx86_64&sha1=3cbe25cde3cfa028984618ad32de9edc4c1eaed0 2023-11-17T05:14:41.370 INFO:teuthology.task.install.rpm:Pulling from https://chacra.ceph.com/r/ceph/pacific/3cbe25cde3cfa028984618ad32de9edc4c1eaed0/centos/8/flavors/default/ 2023-11-17T05:14:41.500 INFO:teuthology.task.install.rpm:Package version is 16.2.4-0 2023-11-17T05:14:41.555 INFO:teuthology.packaging:Writing yum repo: [ceph] name=ceph packages for $basearch baseurl=https://chacra.ceph.com/r/ceph/pacific/3cbe25cde3cfa028984618ad32de9edc4c1eaed0/centos/8/flavors/default/$basearch enabled=1 gpgcheck=0 type=rpm-md [ceph-noarch] name=ceph noarch packages baseurl=https://chacra.ceph.com/r/ceph/pacific/3cbe25cde3cfa028984618ad32de9edc4c1eaed0/centos/8/flavors/default/noarch enabled=1 gpgcheck=0 type=rpm-md [ceph-source] name=ceph source packages baseurl=https://chacra.ceph.com/r/ceph/pacific/3cbe25cde3cfa028984618ad32de9edc4c1eaed0/centos/8/flavors/default/SRPMS enabled=1 gpgcheck=0 type=rpm-md 2023-11-17T05:14:41.555 DEBUG:teuthology.orchestra.run.smithi066:> set -ex 2023-11-17T05:14:41.556 DEBUG:teuthology.orchestra.run.smithi066:> sudo dd of=/etc/yum.repos.d/ceph.repo 2023-11-17T05:14:41.578 INFO:teuthology.task.install.rpm:Pulling from https://chacra.ceph.com/r/ceph/pacific/3cbe25cde3cfa028984618ad32de9edc4c1eaed0/centos/8/flavors/default/ 2023-11-17T05:14:41.578 INFO:teuthology.task.install.rpm:Package version is 16.2.4-0 2023-11-17T05:14:41.595 INFO:teuthology.task.install.rpm:Installing packages: ceph-radosgw, ceph-test, ceph, ceph-base, cephadm, ceph-immutable-object-cache, ceph-mgr, ceph-mgr-dashboard, ceph-mgr-diskprediction-local, ceph-mgr-rook, ceph-mgr-cephadm, ceph-fuse, librados-devel, libcephfs2, libcephfs-devel, librados2, librbd1, python3-rados, python3-rgw, python3-cephfs, python3-rbd, rbd-fuse, rbd-mirror, rbd-nbd on remote rpm x86_64 2023-11-17T05:14:41.595 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using tag 2023-11-17T05:14:41.595 INFO:teuthology.packaging:ref: None 2023-11-17T05:14:41.596 INFO:teuthology.packaging:tag: v16.2.4 2023-11-17T05:14:41.596 INFO:teuthology.packaging:branch: None 2023-11-17T05:14:41.596 INFO:teuthology.packaging:sha1: ed3c8e926b9de93026c8a7c1d86be5d6821b52ac 2023-11-17T05:14:41.597 DEBUG:teuthology.orchestra.run.smithi066:> if test -f /etc/yum.repos.d/ceph.repo ; then sudo sed -i -e ':a;N;$!ba;s/enabled=1\ngpg/enabled=1\npriority=1\ngpg/g' -e 's;ref/[a-zA-Z0-9_-]*/;ref/v16.2.4/;g' /etc/yum.repos.d/ceph.repo ; fi 2023-11-17T05:14:41.616 INFO:teuthology.packaging:Writing yum repo: [ceph] name=ceph packages for $basearch baseurl=https://chacra.ceph.com/r/ceph/pacific/3cbe25cde3cfa028984618ad32de9edc4c1eaed0/centos/8/flavors/default/$basearch enabled=1 gpgcheck=0 type=rpm-md [ceph-noarch] name=ceph noarch packages baseurl=https://chacra.ceph.com/r/ceph/pacific/3cbe25cde3cfa028984618ad32de9edc4c1eaed0/centos/8/flavors/default/noarch enabled=1 gpgcheck=0 type=rpm-md [ceph-source] name=ceph source packages baseurl=https://chacra.ceph.com/r/ceph/pacific/3cbe25cde3cfa028984618ad32de9edc4c1eaed0/centos/8/flavors/default/SRPMS enabled=1 gpgcheck=0 type=rpm-md 2023-11-17T05:14:41.616 DEBUG:teuthology.orchestra.run.smithi040:> set -ex 2023-11-17T05:14:41.616 DEBUG:teuthology.orchestra.run.smithi040:> sudo dd of=/etc/yum.repos.d/ceph.repo 2023-11-17T05:14:41.663 INFO:teuthology.task.install.rpm:Installing packages: ceph-radosgw, ceph-test, ceph, ceph-base, cephadm, ceph-immutable-object-cache, ceph-mgr, ceph-mgr-dashboard, ceph-mgr-diskprediction-local, ceph-mgr-rook, ceph-mgr-cephadm, ceph-fuse, librados-devel, libcephfs2, libcephfs-devel, librados2, librbd1, python3-rados, python3-rgw, python3-cephfs, python3-rbd, rbd-fuse, rbd-mirror, rbd-nbd on remote rpm x86_64 2023-11-17T05:14:41.663 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using tag 2023-11-17T05:14:41.663 INFO:teuthology.packaging:ref: None 2023-11-17T05:14:41.663 INFO:teuthology.packaging:tag: v16.2.4 2023-11-17T05:14:41.663 INFO:teuthology.packaging:branch: None 2023-11-17T05:14:41.663 INFO:teuthology.packaging:sha1: ed3c8e926b9de93026c8a7c1d86be5d6821b52ac 2023-11-17T05:14:41.663 DEBUG:teuthology.orchestra.run.smithi040:> if test -f /etc/yum.repos.d/ceph.repo ; then sudo sed -i -e ':a;N;$!ba;s/enabled=1\ngpg/enabled=1\npriority=1\ngpg/g' -e 's;ref/[a-zA-Z0-9_-]*/;ref/v16.2.4/;g' /etc/yum.repos.d/ceph.repo ; fi 2023-11-17T05:14:41.668 DEBUG:teuthology.orchestra.run.smithi066:> sudo touch -a /etc/yum/pluginconf.d/priorities.conf ; test -e /etc/yum/pluginconf.d/priorities.conf.orig || sudo cp -af /etc/yum/pluginconf.d/priorities.conf /etc/yum/pluginconf.d/priorities.conf.orig 2023-11-17T05:14:41.740 DEBUG:teuthology.orchestra.run.smithi040:> sudo touch -a /etc/yum/pluginconf.d/priorities.conf ; test -e /etc/yum/pluginconf.d/priorities.conf.orig || sudo cp -af /etc/yum/pluginconf.d/priorities.conf /etc/yum/pluginconf.d/priorities.conf.orig 2023-11-17T05:14:41.749 DEBUG:teuthology.orchestra.run.smithi066:> grep check_obsoletes /etc/yum/pluginconf.d/priorities.conf && sudo sed -i 's/check_obsoletes.*0/check_obsoletes = 1/g' /etc/yum/pluginconf.d/priorities.conf || echo 'check_obsoletes = 1' | sudo tee -a /etc/yum/pluginconf.d/priorities.conf 2023-11-17T05:14:41.818 INFO:teuthology.orchestra.run.smithi066.stdout:check_obsoletes = 1 2023-11-17T05:14:41.819 DEBUG:teuthology.orchestra.run.smithi066:> sudo yum clean all 2023-11-17T05:14:41.826 DEBUG:teuthology.orchestra.run.smithi040:> grep check_obsoletes /etc/yum/pluginconf.d/priorities.conf && sudo sed -i 's/check_obsoletes.*0/check_obsoletes = 1/g' /etc/yum/pluginconf.d/priorities.conf || echo 'check_obsoletes = 1' | sudo tee -a /etc/yum/pluginconf.d/priorities.conf 2023-11-17T05:14:41.858 INFO:teuthology.orchestra.run.smithi040.stdout:check_obsoletes = 1 2023-11-17T05:14:41.863 DEBUG:teuthology.orchestra.run.smithi040:> sudo yum clean all 2023-11-17T05:14:42.109 INFO:teuthology.orchestra.run.smithi066.stdout:76 files removed 2023-11-17T05:14:42.141 DEBUG:teuthology.orchestra.run.smithi066:> sudo yum -y install ceph-radosgw 2023-11-17T05:14:42.172 INFO:teuthology.orchestra.run.smithi040.stdout:76 files removed 2023-11-17T05:14:42.205 DEBUG:teuthology.orchestra.run.smithi040:> sudo yum -y install ceph-radosgw 2023-11-17T05:14:42.652 INFO:teuthology.orchestra.run.smithi066.stdout:ceph packages for x86_64 900 kB/s | 74 kB 00:00 2023-11-17T05:14:42.768 INFO:teuthology.orchestra.run.smithi066.stdout:ceph noarch packages 198 kB/s | 15 kB 00:00 2023-11-17T05:14:42.792 INFO:teuthology.orchestra.run.smithi040.stdout:ceph packages for x86_64 626 kB/s | 74 kB 00:00 2023-11-17T05:14:42.886 INFO:teuthology.orchestra.run.smithi066.stdout:ceph source packages 20 kB/s | 1.7 kB 00:00 2023-11-17T05:14:42.918 INFO:teuthology.orchestra.run.smithi040.stdout:ceph noarch packages 184 kB/s | 15 kB 00:00 2023-11-17T05:14:43.026 INFO:teuthology.orchestra.run.smithi040.stdout:ceph source packages 23 kB/s | 1.7 kB 00:00 2023-11-17T05:14:44.635 INFO:teuthology.orchestra.run.smithi040.stdout:CentOS Stream 8 - AppStream 22 MB/s | 34 MB 00:01 2023-11-17T05:14:46.069 INFO:teuthology.orchestra.run.smithi066.stdout:CentOS Stream 8 - AppStream 11 MB/s | 34 MB 00:03 2023-11-17T05:14:53.793 INFO:teuthology.orchestra.run.smithi040.stdout:CentOS Stream 8 - BaseOS 25 MB/s | 53 MB 00:02 2023-11-17T05:14:54.945 INFO:teuthology.orchestra.run.smithi066.stdout:CentOS Stream 8 - BaseOS 25 MB/s | 53 MB 00:02 2023-11-17T05:15:02.460 INFO:teuthology.orchestra.run.smithi040.stdout:CentOS Stream 8 - Extras 60 kB/s | 18 kB 00:00 2023-11-17T05:15:04.479 INFO:teuthology.orchestra.run.smithi066.stdout:CentOS Stream 8 - Extras 12 kB/s | 18 kB 00:01 2023-11-17T05:15:05.486 INFO:teuthology.orchestra.run.smithi066.stdout:CentOS Stream 8 - Extras common packages 8.6 kB/s | 6.9 kB 00:00 2023-11-17T05:15:06.713 INFO:teuthology.orchestra.run.smithi066.stdout:CentOS Stream 8 - PowerTools 6.3 MB/s | 6.6 MB 00:01 2023-11-17T05:15:07.068 INFO:teuthology.orchestra.run.smithi040.stdout:CentOS Stream 8 - Extras common packages 1.6 kB/s | 6.9 kB 00:04 2023-11-17T05:15:08.197 INFO:teuthology.orchestra.run.smithi066.stdout:Copr repo for python3-asyncssh owned by ceph 31 kB/s | 3.5 kB 00:00 2023-11-17T05:15:09.359 INFO:teuthology.orchestra.run.smithi066.stdout:Extra Packages for Enterprise Linux 16 MB/s | 16 MB 00:00 2023-11-17T05:15:09.544 INFO:teuthology.orchestra.run.smithi040.stdout:CentOS Stream 8 - PowerTools 2.9 MB/s | 6.6 MB 00:02 2023-11-17T05:15:11.121 INFO:teuthology.orchestra.run.smithi040.stdout:Copr repo for python3-asyncssh owned by ceph 24 kB/s | 3.5 kB 00:00 2023-11-17T05:15:13.408 INFO:teuthology.orchestra.run.smithi066.stdout:lab-extras 433 kB/s | 24 kB 00:00 2023-11-17T05:15:14.750 INFO:teuthology.orchestra.run.smithi040.stdout:Extra Packages for Enterprise Linux 4.6 MB/s | 16 MB 00:03 2023-11-17T05:15:18.294 INFO:teuthology.orchestra.run.smithi066.stdout:Dependencies resolved. 2023-11-17T05:15:18.298 INFO:teuthology.orchestra.run.smithi066.stdout:================================================================================ 2023-11-17T05:15:18.298 INFO:teuthology.orchestra.run.smithi066.stdout: Package Arch Version Repository Size 2023-11-17T05:15:18.298 INFO:teuthology.orchestra.run.smithi066.stdout:================================================================================ 2023-11-17T05:15:18.298 INFO:teuthology.orchestra.run.smithi066.stdout:Installing: 2023-11-17T05:15:18.298 INFO:teuthology.orchestra.run.smithi066.stdout: ceph-radosgw x86_64 2:16.2.4-0.el8 ceph 13 M 2023-11-17T05:15:18.298 INFO:teuthology.orchestra.run.smithi066.stdout:Upgrading: 2023-11-17T05:15:18.298 INFO:teuthology.orchestra.run.smithi066.stdout: librados2 x86_64 2:16.2.4-0.el8 ceph 3.7 M 2023-11-17T05:15:18.298 INFO:teuthology.orchestra.run.smithi066.stdout: librbd1 x86_64 2:16.2.4-0.el8 ceph 4.0 M 2023-11-17T05:15:18.298 INFO:teuthology.orchestra.run.smithi066.stdout:Installing dependencies: 2023-11-17T05:15:18.299 INFO:teuthology.orchestra.run.smithi066.stdout: ceph-base x86_64 2:16.2.4-0.el8 ceph 6.0 M 2023-11-17T05:15:18.299 INFO:teuthology.orchestra.run.smithi066.stdout: ceph-common x86_64 2:16.2.4-0.el8 ceph 24 M 2023-11-17T05:15:18.299 INFO:teuthology.orchestra.run.smithi066.stdout: ceph-selinux x86_64 2:16.2.4-0.el8 ceph 25 k 2023-11-17T05:15:18.299 INFO:teuthology.orchestra.run.smithi066.stdout: daxctl-libs x86_64 71.1-7.el8 baseos 42 k 2023-11-17T05:15:18.299 INFO:teuthology.orchestra.run.smithi066.stdout: fmt x86_64 6.2.1-1.el8 epel 116 k 2023-11-17T05:15:18.299 INFO:teuthology.orchestra.run.smithi066.stdout: gperftools-libs x86_64 1:2.7-9.el8 epel 306 k 2023-11-17T05:15:18.299 INFO:teuthology.orchestra.run.smithi066.stdout: leveldb x86_64 1.22-1.el8 epel 181 k 2023-11-17T05:15:18.299 INFO:teuthology.orchestra.run.smithi066.stdout: libcephfs2 x86_64 2:16.2.4-0.el8 ceph 801 k 2023-11-17T05:15:18.299 INFO:teuthology.orchestra.run.smithi066.stdout: liboath x86_64 2.6.2-3.el8 epel 59 k 2023-11-17T05:15:18.299 INFO:teuthology.orchestra.run.smithi066.stdout: libpmemobj x86_64 1.6.1-1.el8 appstream 145 k 2023-11-17T05:15:18.299 INFO:teuthology.orchestra.run.smithi066.stdout: librabbitmq x86_64 0.9.0-4.el8 baseos 47 k 2023-11-17T05:15:18.299 INFO:teuthology.orchestra.run.smithi066.stdout: libradosstriper1 x86_64 2:16.2.4-0.el8 ceph 486 k 2023-11-17T05:15:18.300 INFO:teuthology.orchestra.run.smithi066.stdout: librdkafka x86_64 0.11.4-3.el8 appstream 354 k 2023-11-17T05:15:18.300 INFO:teuthology.orchestra.run.smithi066.stdout: librgw2 x86_64 2:16.2.4-0.el8 ceph 4.1 M 2023-11-17T05:15:18.300 INFO:teuthology.orchestra.run.smithi066.stdout: libunwind x86_64 1.3.1-3.el8 epel 75 k 2023-11-17T05:15:18.300 INFO:teuthology.orchestra.run.smithi066.stdout: lttng-ust x86_64 2.8.1-11.el8 appstream 259 k 2023-11-17T05:15:18.300 INFO:teuthology.orchestra.run.smithi066.stdout: ndctl-libs x86_64 71.1-7.el8 baseos 80 k 2023-11-17T05:15:18.300 INFO:teuthology.orchestra.run.smithi066.stdout: python3-ceph-argparse x86_64 2:16.2.4-0.el8 ceph 44 k 2023-11-17T05:15:18.300 INFO:teuthology.orchestra.run.smithi066.stdout: python3-ceph-common x86_64 2:16.2.4-0.el8 ceph 80 k 2023-11-17T05:15:18.300 INFO:teuthology.orchestra.run.smithi066.stdout: python3-cephfs x86_64 2:16.2.4-0.el8 ceph 213 k 2023-11-17T05:15:18.300 INFO:teuthology.orchestra.run.smithi066.stdout: python3-prettytable noarch 0.7.2-14.el8 appstream 44 k 2023-11-17T05:15:18.300 INFO:teuthology.orchestra.run.smithi066.stdout: python3-rados x86_64 2:16.2.4-0.el8 ceph 387 k 2023-11-17T05:15:18.300 INFO:teuthology.orchestra.run.smithi066.stdout: python3-rbd x86_64 2:16.2.4-0.el8 ceph 368 k 2023-11-17T05:15:18.300 INFO:teuthology.orchestra.run.smithi066.stdout: python3-rgw x86_64 2:16.2.4-0.el8 ceph 114 k 2023-11-17T05:15:18.301 INFO:teuthology.orchestra.run.smithi066.stdout: 2023-11-17T05:15:18.301 INFO:teuthology.orchestra.run.smithi066.stdout:Transaction Summary 2023-11-17T05:15:18.301 INFO:teuthology.orchestra.run.smithi066.stdout:================================================================================ 2023-11-17T05:15:18.301 INFO:teuthology.orchestra.run.smithi066.stdout:Install 25 Packages 2023-11-17T05:15:18.301 INFO:teuthology.orchestra.run.smithi066.stdout:Upgrade 2 Packages 2023-11-17T05:15:18.301 INFO:teuthology.orchestra.run.smithi066.stdout: 2023-11-17T05:15:18.301 INFO:teuthology.orchestra.run.smithi066.stdout:Total download size: 59 M 2023-11-17T05:15:18.301 INFO:teuthology.orchestra.run.smithi066.stdout:Downloading Packages: 2023-11-17T05:15:18.583 INFO:teuthology.orchestra.run.smithi066.stdout:(1/27): ceph-base-16.2.4-0.el8.x86_64.rpm 34 MB/s | 6.0 MB 00:00 2023-11-17T05:15:18.608 INFO:teuthology.orchestra.run.smithi066.stdout:(2/27): ceph-selinux-16.2.4-0.el8.x86_64.rpm 995 kB/s | 25 kB 00:00 2023-11-17T05:15:18.675 INFO:teuthology.orchestra.run.smithi066.stdout:(3/27): libcephfs2-16.2.4-0.el8.x86_64.rpm 12 MB/s | 801 kB 00:00 2023-11-17T05:15:18.709 INFO:teuthology.orchestra.run.smithi066.stdout:(4/27): libradosstriper1-16.2.4-0.el8.x86_64.rp 14 MB/s | 486 kB 00:00 2023-11-17T05:15:18.860 INFO:teuthology.orchestra.run.smithi066.stdout:(5/27): ceph-radosgw-16.2.4-0.el8.x86_64.rpm 28 MB/s | 13 MB 00:00 2023-11-17T05:15:18.884 INFO:teuthology.orchestra.run.smithi066.stdout:(6/27): python3-ceph-argparse-16.2.4-0.el8.x86_ 1.8 MB/s | 44 kB 00:00 2023-11-17T05:15:18.911 INFO:teuthology.orchestra.run.smithi066.stdout:(7/27): python3-ceph-common-16.2.4-0.el8.x86_64 3.1 MB/s | 80 kB 00:00 2023-11-17T05:15:18.935 INFO:teuthology.orchestra.run.smithi066.stdout:(8/27): python3-cephfs-16.2.4-0.el8.x86_64.rpm 8.4 MB/s | 213 kB 00:00 2023-11-17T05:15:18.969 INFO:teuthology.orchestra.run.smithi066.stdout:(9/27): python3-rados-16.2.4-0.el8.x86_64.rpm 11 MB/s | 387 kB 00:00 2023-11-17T05:15:19.002 INFO:teuthology.orchestra.run.smithi066.stdout:(10/27): python3-rbd-16.2.4-0.el8.x86_64.rpm 11 MB/s | 368 kB 00:00 2023-11-17T05:15:19.028 INFO:teuthology.orchestra.run.smithi066.stdout:(11/27): python3-rgw-16.2.4-0.el8.x86_64.rpm 4.5 MB/s | 114 kB 00:00 2023-11-17T05:15:19.030 INFO:teuthology.orchestra.run.smithi040.stdout:lab-extras 424 kB/s | 24 kB 00:00 2023-11-17T05:15:19.095 INFO:teuthology.orchestra.run.smithi066.stdout:(12/27): librgw2-16.2.4-0.el8.x86_64.rpm 11 MB/s | 4.1 MB 00:00 2023-11-17T05:15:19.370 INFO:teuthology.orchestra.run.smithi066.stdout:(13/27): ceph-common-16.2.4-0.el8.x86_64.rpm 25 MB/s | 24 MB 00:00 2023-11-17T05:15:19.395 INFO:teuthology.orchestra.run.smithi066.stdout:(14/27): libpmemobj-1.6.1-1.el8.x86_64.rpm 395 kB/s | 145 kB 00:00 2023-11-17T05:15:19.454 INFO:teuthology.orchestra.run.smithi066.stdout:(15/27): python3-prettytable-0.7.2-14.el8.noarc 747 kB/s | 44 kB 00:00 2023-11-17T05:15:19.496 INFO:teuthology.orchestra.run.smithi066.stdout:(16/27): librdkafka-0.11.4-3.el8.x86_64.rpm 882 kB/s | 354 kB 00:00 2023-11-17T05:15:19.571 INFO:teuthology.orchestra.run.smithi066.stdout:(17/27): daxctl-libs-71.1-7.el8.x86_64.rpm 361 kB/s | 42 kB 00:00 2023-11-17T05:15:19.600 INFO:teuthology.orchestra.run.smithi066.stdout:(18/27): librabbitmq-0.9.0-4.el8.x86_64.rpm 467 kB/s | 47 kB 00:00 2023-11-17T05:15:19.647 INFO:teuthology.orchestra.run.smithi066.stdout:(19/27): ndctl-libs-71.1-7.el8.x86_64.rpm 1.0 MB/s | 80 kB 00:00 2023-11-17T05:15:19.664 INFO:teuthology.orchestra.run.smithi066.stdout:(20/27): lttng-ust-2.8.1-11.el8.x86_64.rpm 885 kB/s | 259 kB 00:00 2023-11-17T05:15:20.198 INFO:teuthology.orchestra.run.smithi066.stdout:(21/27): fmt-6.2.1-1.el8.x86_64.rpm 194 kB/s | 116 kB 00:00 2023-11-17T05:15:20.256 INFO:teuthology.orchestra.run.smithi066.stdout:(22/27): leveldb-1.22-1.el8.x86_64.rpm 306 kB/s | 181 kB 00:00 2023-11-17T05:15:20.306 INFO:teuthology.orchestra.run.smithi066.stdout:(23/27): liboath-2.6.2-3.el8.x86_64.rpm 542 kB/s | 59 kB 00:00 2023-11-17T05:15:20.382 INFO:teuthology.orchestra.run.smithi066.stdout:(24/27): librados2-16.2.4-0.el8.x86_64.rpm 50 MB/s | 3.7 MB 00:00 2023-11-17T05:15:20.407 INFO:teuthology.orchestra.run.smithi066.stdout:(25/27): gperftools-libs-2.7-9.el8.x86_64.rpm 403 kB/s | 306 kB 00:00 2023-11-17T05:15:20.432 INFO:teuthology.orchestra.run.smithi066.stdout:(26/27): libunwind-1.3.1-3.el8.x86_64.rpm 427 kB/s | 75 kB 00:00 2023-11-17T05:15:20.532 INFO:teuthology.orchestra.run.smithi066.stdout:(27/27): librbd1-16.2.4-0.el8.x86_64.rpm 27 MB/s | 4.0 MB 00:00 2023-11-17T05:15:20.536 INFO:teuthology.orchestra.run.smithi066.stdout:-------------------------------------------------------------------------------- 2023-11-17T05:15:20.537 INFO:teuthology.orchestra.run.smithi066.stdout:Total 26 MB/s | 59 MB 00:02 2023-11-17T05:15:20.611 INFO:teuthology.orchestra.run.smithi066.stdout:Running transaction check 2023-11-17T05:15:20.659 INFO:teuthology.orchestra.run.smithi066.stdout:Transaction check succeeded. 2023-11-17T05:15:20.659 INFO:teuthology.orchestra.run.smithi066.stdout:Running transaction test 2023-11-17T05:15:21.020 INFO:teuthology.orchestra.run.smithi066.stdout:Transaction test succeeded. 2023-11-17T05:15:21.023 INFO:teuthology.orchestra.run.smithi066.stdout:Running transaction 2023-11-17T05:15:22.187 INFO:teuthology.orchestra.run.smithi066.stdout: Preparing : 1/1 2023-11-17T05:15:22.382 INFO:teuthology.orchestra.run.smithi066.stdout: Installing : fmt-6.2.1-1.el8.x86_64 1/29 2023-11-17T05:15:22.588 INFO:teuthology.orchestra.run.smithi066.stdout: Installing : liboath-2.6.2-3.el8.x86_64 2/29 2023-11-17T05:15:22.738 INFO:teuthology.orchestra.run.smithi066.stdout: Installing : lttng-ust-2.8.1-11.el8.x86_64 3/29 2023-11-17T05:15:23.161 INFO:teuthology.orchestra.run.smithi066.stdout: Running scriptlet: lttng-ust-2.8.1-11.el8.x86_64 3/29 2023-11-17T05:15:23.339 INFO:teuthology.orchestra.run.smithi066.stdout: Upgrading : librados2-2:16.2.4-0.el8.x86_64 4/29 2023-11-17T05:15:23.443 INFO:teuthology.orchestra.run.smithi066.stdout: Running scriptlet: librados2-2:16.2.4-0.el8.x86_64 4/29 2023-11-17T05:15:23.686 INFO:teuthology.orchestra.run.smithi066.stdout: Installing : python3-rados-2:16.2.4-0.el8.x86_64 5/29 2023-11-17T05:15:23.822 INFO:teuthology.orchestra.run.smithi066.stdout: Installing : libcephfs2-2:16.2.4-0.el8.x86_64 6/29 2023-11-17T05:15:23.889 INFO:teuthology.orchestra.run.smithi066.stdout: Running scriptlet: libcephfs2-2:16.2.4-0.el8.x86_64 6/29 2023-11-17T05:15:24.105 INFO:teuthology.orchestra.run.smithi066.stdout: Installing : librabbitmq-0.9.0-4.el8.x86_64 7/29 2023-11-17T05:15:24.193 INFO:teuthology.orchestra.run.smithi040.stdout:Dependencies resolved. 2023-11-17T05:15:24.197 INFO:teuthology.orchestra.run.smithi040.stdout:================================================================================ 2023-11-17T05:15:24.197 INFO:teuthology.orchestra.run.smithi040.stdout: Package Arch Version Repository Size 2023-11-17T05:15:24.197 INFO:teuthology.orchestra.run.smithi040.stdout:================================================================================ 2023-11-17T05:15:24.197 INFO:teuthology.orchestra.run.smithi040.stdout:Installing: 2023-11-17T05:15:24.197 INFO:teuthology.orchestra.run.smithi040.stdout: ceph-radosgw x86_64 2:16.2.4-0.el8 ceph 13 M 2023-11-17T05:15:24.197 INFO:teuthology.orchestra.run.smithi040.stdout:Upgrading: 2023-11-17T05:15:24.197 INFO:teuthology.orchestra.run.smithi040.stdout: librados2 x86_64 2:16.2.4-0.el8 ceph 3.7 M 2023-11-17T05:15:24.197 INFO:teuthology.orchestra.run.smithi040.stdout: librbd1 x86_64 2:16.2.4-0.el8 ceph 4.0 M 2023-11-17T05:15:24.197 INFO:teuthology.orchestra.run.smithi040.stdout:Installing dependencies: 2023-11-17T05:15:24.197 INFO:teuthology.orchestra.run.smithi040.stdout: ceph-base x86_64 2:16.2.4-0.el8 ceph 6.0 M 2023-11-17T05:15:24.198 INFO:teuthology.orchestra.run.smithi040.stdout: ceph-common x86_64 2:16.2.4-0.el8 ceph 24 M 2023-11-17T05:15:24.198 INFO:teuthology.orchestra.run.smithi040.stdout: ceph-selinux x86_64 2:16.2.4-0.el8 ceph 25 k 2023-11-17T05:15:24.198 INFO:teuthology.orchestra.run.smithi040.stdout: daxctl-libs x86_64 71.1-7.el8 baseos 42 k 2023-11-17T05:15:24.198 INFO:teuthology.orchestra.run.smithi040.stdout: fmt x86_64 6.2.1-1.el8 epel 116 k 2023-11-17T05:15:24.198 INFO:teuthology.orchestra.run.smithi040.stdout: gperftools-libs x86_64 1:2.7-9.el8 epel 306 k 2023-11-17T05:15:24.198 INFO:teuthology.orchestra.run.smithi040.stdout: leveldb x86_64 1.22-1.el8 epel 181 k 2023-11-17T05:15:24.198 INFO:teuthology.orchestra.run.smithi040.stdout: libcephfs2 x86_64 2:16.2.4-0.el8 ceph 801 k 2023-11-17T05:15:24.198 INFO:teuthology.orchestra.run.smithi040.stdout: liboath x86_64 2.6.2-3.el8 epel 59 k 2023-11-17T05:15:24.198 INFO:teuthology.orchestra.run.smithi040.stdout: libpmemobj x86_64 1.6.1-1.el8 appstream 145 k 2023-11-17T05:15:24.198 INFO:teuthology.orchestra.run.smithi040.stdout: librabbitmq x86_64 0.9.0-4.el8 baseos 47 k 2023-11-17T05:15:24.198 INFO:teuthology.orchestra.run.smithi040.stdout: libradosstriper1 x86_64 2:16.2.4-0.el8 ceph 486 k 2023-11-17T05:15:24.198 INFO:teuthology.orchestra.run.smithi040.stdout: librdkafka x86_64 0.11.4-3.el8 appstream 354 k 2023-11-17T05:15:24.199 INFO:teuthology.orchestra.run.smithi040.stdout: librgw2 x86_64 2:16.2.4-0.el8 ceph 4.1 M 2023-11-17T05:15:24.199 INFO:teuthology.orchestra.run.smithi040.stdout: libunwind x86_64 1.3.1-3.el8 epel 75 k 2023-11-17T05:15:24.199 INFO:teuthology.orchestra.run.smithi040.stdout: lttng-ust x86_64 2.8.1-11.el8 appstream 259 k 2023-11-17T05:15:24.199 INFO:teuthology.orchestra.run.smithi040.stdout: ndctl-libs x86_64 71.1-7.el8 baseos 80 k 2023-11-17T05:15:24.199 INFO:teuthology.orchestra.run.smithi040.stdout: python3-ceph-argparse x86_64 2:16.2.4-0.el8 ceph 44 k 2023-11-17T05:15:24.199 INFO:teuthology.orchestra.run.smithi040.stdout: python3-ceph-common x86_64 2:16.2.4-0.el8 ceph 80 k 2023-11-17T05:15:24.199 INFO:teuthology.orchestra.run.smithi040.stdout: python3-cephfs x86_64 2:16.2.4-0.el8 ceph 213 k 2023-11-17T05:15:24.199 INFO:teuthology.orchestra.run.smithi040.stdout: python3-prettytable noarch 0.7.2-14.el8 appstream 44 k 2023-11-17T05:15:24.199 INFO:teuthology.orchestra.run.smithi040.stdout: python3-rados x86_64 2:16.2.4-0.el8 ceph 387 k 2023-11-17T05:15:24.199 INFO:teuthology.orchestra.run.smithi040.stdout: python3-rbd x86_64 2:16.2.4-0.el8 ceph 368 k 2023-11-17T05:15:24.199 INFO:teuthology.orchestra.run.smithi040.stdout: python3-rgw x86_64 2:16.2.4-0.el8 ceph 114 k 2023-11-17T05:15:24.199 INFO:teuthology.orchestra.run.smithi040.stdout: 2023-11-17T05:15:24.200 INFO:teuthology.orchestra.run.smithi040.stdout:Transaction Summary 2023-11-17T05:15:24.200 INFO:teuthology.orchestra.run.smithi040.stdout:================================================================================ 2023-11-17T05:15:24.200 INFO:teuthology.orchestra.run.smithi040.stdout:Install 25 Packages 2023-11-17T05:15:24.200 INFO:teuthology.orchestra.run.smithi040.stdout:Upgrade 2 Packages 2023-11-17T05:15:24.200 INFO:teuthology.orchestra.run.smithi040.stdout: 2023-11-17T05:15:24.200 INFO:teuthology.orchestra.run.smithi040.stdout:Total download size: 59 M 2023-11-17T05:15:24.200 INFO:teuthology.orchestra.run.smithi040.stdout:Downloading Packages: 2023-11-17T05:15:24.280 INFO:teuthology.orchestra.run.smithi066.stdout: Installing : librdkafka-0.11.4-3.el8.x86_64 8/29 2023-11-17T05:15:24.725 INFO:teuthology.orchestra.run.smithi066.stdout: Running scriptlet: librdkafka-0.11.4-3.el8.x86_64 8/29 2023-11-17T05:15:24.907 INFO:teuthology.orchestra.run.smithi040.stdout:(1/27): ceph-base-16.2.4-0.el8.x86_64.rpm 37 MB/s | 6.0 MB 00:00 2023-11-17T05:15:24.932 INFO:teuthology.orchestra.run.smithi040.stdout:(2/27): ceph-selinux-16.2.4-0.el8.x86_64.rpm 1.0 MB/s | 25 kB 00:00 2023-11-17T05:15:24.974 INFO:teuthology.orchestra.run.smithi040.stdout:(3/27): libcephfs2-16.2.4-0.el8.x86_64.rpm 19 MB/s | 801 kB 00:00 2023-11-17T05:15:25.007 INFO:teuthology.orchestra.run.smithi040.stdout:(4/27): libradosstriper1-16.2.4-0.el8.x86_64.rp 14 MB/s | 486 kB 00:00 2023-11-17T05:15:25.080 INFO:teuthology.orchestra.run.smithi066.stdout: Installing : librgw2-2:16.2.4-0.el8.x86_64 9/29 2023-11-17T05:15:25.107 INFO:teuthology.orchestra.run.smithi040.stdout:(5/27): librgw2-16.2.4-0.el8.x86_64.rpm 41 MB/s | 4.1 MB 00:00 2023-11-17T05:15:25.161 INFO:teuthology.orchestra.run.smithi040.stdout:(6/27): python3-ceph-argparse-16.2.4-0.el8.x86_ 865 kB/s | 44 kB 00:00 2023-11-17T05:15:25.162 INFO:teuthology.orchestra.run.smithi066.stdout: Running scriptlet: librgw2-2:16.2.4-0.el8.x86_64 9/29 2023-11-17T05:15:25.300 INFO:teuthology.orchestra.run.smithi040.stdout:(7/27): python3-ceph-common-16.2.4-0.el8.x86_64 570 kB/s | 80 kB 00:00 2023-11-17T05:15:25.325 INFO:teuthology.orchestra.run.smithi040.stdout:(8/27): python3-cephfs-16.2.4-0.el8.x86_64.rpm 8.4 MB/s | 213 kB 00:00 2023-11-17T05:15:25.353 INFO:teuthology.orchestra.run.smithi066.stdout: Installing : leveldb-1.22-1.el8.x86_64 10/29 2023-11-17T05:15:25.359 INFO:teuthology.orchestra.run.smithi040.stdout:(9/27): python3-rados-16.2.4-0.el8.x86_64.rpm 11 MB/s | 387 kB 00:00 2023-11-17T05:15:25.392 INFO:teuthology.orchestra.run.smithi040.stdout:(10/27): python3-rbd-16.2.4-0.el8.x86_64.rpm 11 MB/s | 368 kB 00:00 2023-11-17T05:15:25.418 INFO:teuthology.orchestra.run.smithi040.stdout:(11/27): python3-rgw-16.2.4-0.el8.x86_64.rpm 4.5 MB/s | 114 kB 00:00 2023-11-17T05:15:25.555 INFO:teuthology.orchestra.run.smithi066.stdout: Installing : daxctl-libs-71.1-7.el8.x86_64 11/29 2023-11-17T05:15:25.586 INFO:teuthology.orchestra.run.smithi040.stdout:(12/27): ceph-radosgw-16.2.4-0.el8.x86_64.rpm 15 MB/s | 13 MB 00:00 2023-11-17T05:15:25.627 INFO:teuthology.orchestra.run.smithi066.stdout: Running scriptlet: daxctl-libs-71.1-7.el8.x86_64 11/29 2023-11-17T05:15:25.789 INFO:teuthology.orchestra.run.smithi066.stdout: Installing : python3-ceph-argparse-2:16.2.4-0.el8.x86_64 12/29 2023-11-17T05:15:25.893 INFO:teuthology.orchestra.run.smithi040.stdout:(13/27): ceph-common-16.2.4-0.el8.x86_64.rpm 21 MB/s | 24 MB 00:01 2023-11-17T05:15:25.919 INFO:teuthology.orchestra.run.smithi040.stdout:(14/27): libpmemobj-1.6.1-1.el8.x86_64.rpm 290 kB/s | 145 kB 00:00 2023-11-17T05:15:25.948 INFO:teuthology.orchestra.run.smithi066.stdout: Installing : python3-cephfs-2:16.2.4-0.el8.x86_64 13/29 2023-11-17T05:15:25.986 INFO:teuthology.orchestra.run.smithi040.stdout:(15/27): python3-prettytable-0.7.2-14.el8.noarc 659 kB/s | 44 kB 00:00 2023-11-17T05:15:26.011 INFO:teuthology.orchestra.run.smithi040.stdout:(16/27): librdkafka-0.11.4-3.el8.x86_64.rpm 831 kB/s | 354 kB 00:00 2023-11-17T05:15:26.111 INFO:teuthology.orchestra.run.smithi040.stdout:(17/27): daxctl-libs-71.1-7.el8.x86_64.rpm 337 kB/s | 42 kB 00:00 2023-11-17T05:15:26.137 INFO:teuthology.orchestra.run.smithi040.stdout:(18/27): librabbitmq-0.9.0-4.el8.x86_64.rpm 375 kB/s | 47 kB 00:00 2023-11-17T05:15:26.155 INFO:teuthology.orchestra.run.smithi066.stdout: Installing : ndctl-libs-71.1-7.el8.x86_64 14/29 2023-11-17T05:15:26.170 INFO:teuthology.orchestra.run.smithi040.stdout:(19/27): lttng-ust-2.8.1-11.el8.x86_64.rpm 940 kB/s | 259 kB 00:00 2023-11-17T05:15:26.196 INFO:teuthology.orchestra.run.smithi040.stdout:(20/27): ndctl-libs-71.1-7.el8.x86_64.rpm 1.3 MB/s | 80 kB 00:00 2023-11-17T05:15:26.221 INFO:teuthology.orchestra.run.smithi066.stdout: Running scriptlet: ndctl-libs-71.1-7.el8.x86_64 14/29 2023-11-17T05:15:26.363 INFO:teuthology.orchestra.run.smithi066.stdout: Installing : libpmemobj-1.6.1-1.el8.x86_64 15/29 2023-11-17T05:15:26.497 INFO:teuthology.orchestra.run.smithi040.stdout:(21/27): gperftools-libs-2.7-9.el8.x86_64.rpm 941 kB/s | 306 kB 00:00 2023-11-17T05:15:26.566 INFO:teuthology.orchestra.run.smithi040.stdout:(22/27): liboath-2.6.2-3.el8.x86_64.rpm 841 kB/s | 59 kB 00:00 2023-11-17T05:15:26.705 INFO:teuthology.orchestra.run.smithi040.stdout:(23/27): libunwind-1.3.1-3.el8.x86_64.rpm 542 kB/s | 75 kB 00:00 2023-11-17T05:15:26.780 INFO:teuthology.orchestra.run.smithi040.stdout:(24/27): librados2-16.2.4-0.el8.x86_64.rpm 50 MB/s | 3.7 MB 00:00 2023-11-17T05:15:26.802 INFO:teuthology.orchestra.run.smithi066.stdout: Running scriptlet: libpmemobj-1.6.1-1.el8.x86_64 15/29 2023-11-17T05:15:26.856 INFO:teuthology.orchestra.run.smithi040.stdout:(25/27): librbd1-16.2.4-0.el8.x86_64.rpm 54 MB/s | 4.0 MB 00:00 2023-11-17T05:15:26.971 INFO:teuthology.orchestra.run.smithi066.stdout: Upgrading : librbd1-2:16.2.4-0.el8.x86_64 16/29 2023-11-17T05:15:27.097 INFO:teuthology.orchestra.run.smithi066.stdout: Running scriptlet: librbd1-2:16.2.4-0.el8.x86_64 16/29 2023-11-17T05:15:27.259 INFO:teuthology.orchestra.run.smithi066.stdout: Installing : python3-rbd-2:16.2.4-0.el8.x86_64 17/29 2023-11-17T05:15:27.451 INFO:teuthology.orchestra.run.smithi066.stdout: Installing : python3-rgw-2:16.2.4-0.el8.x86_64 18/29 2023-11-17T05:15:27.631 INFO:teuthology.orchestra.run.smithi066.stdout: Installing : libradosstriper1-2:16.2.4-0.el8.x86_64 19/29 2023-11-17T05:15:27.696 INFO:teuthology.orchestra.run.smithi066.stdout: Running scriptlet: libradosstriper1-2:16.2.4-0.el8.x86_64 19/29 2023-11-17T05:15:27.882 INFO:teuthology.orchestra.run.smithi066.stdout: Installing : libunwind-1.3.1-3.el8.x86_64 20/29 2023-11-17T05:15:28.080 INFO:teuthology.orchestra.run.smithi066.stdout: Installing : gperftools-libs-1:2.7-9.el8.x86_64 21/29 2023-11-17T05:15:28.265 INFO:teuthology.orchestra.run.smithi066.stdout: Installing : python3-prettytable-0.7.2-14.el8.noarch 22/29 2023-11-17T05:15:28.455 INFO:teuthology.orchestra.run.smithi066.stdout: Installing : python3-ceph-common-2:16.2.4-0.el8.x86_64 23/29 2023-11-17T05:15:29.688 INFO:teuthology.orchestra.run.smithi040.stdout:(26/27): leveldb-1.22-1.el8.x86_64.rpm 52 kB/s | 181 kB 00:03 2023-11-17T05:15:31.174 INFO:teuthology.orchestra.run.smithi066.stdout: Running scriptlet: ceph-common-2:16.2.4-0.el8.x86_64 24/29 2023-11-17T05:15:31.257 INFO:teuthology.orchestra.run.smithi040.stdout:(27/27): fmt-6.2.1-1.el8.x86_64.rpm 23 kB/s | 116 kB 00:05 2023-11-17T05:15:31.257 INFO:teuthology.orchestra.run.smithi040.stdout:-------------------------------------------------------------------------------- 2023-11-17T05:15:31.258 INFO:teuthology.orchestra.run.smithi040.stdout:Total 8.3 MB/s | 59 MB 00:07 2023-11-17T05:15:31.330 INFO:teuthology.orchestra.run.smithi040.stdout:Running transaction check 2023-11-17T05:15:31.372 INFO:teuthology.orchestra.run.smithi066.stdout: Installing : ceph-common-2:16.2.4-0.el8.x86_64 24/29 2023-11-17T05:15:31.383 INFO:teuthology.orchestra.run.smithi040.stdout:Transaction check succeeded. 2023-11-17T05:15:31.383 INFO:teuthology.orchestra.run.smithi040.stdout:Running transaction test 2023-11-17T05:15:31.768 INFO:teuthology.orchestra.run.smithi040.stdout:Transaction test succeeded. 2023-11-17T05:15:31.772 INFO:teuthology.orchestra.run.smithi040.stdout:Running transaction 2023-11-17T05:15:32.109 INFO:teuthology.orchestra.run.smithi066.stdout: Running scriptlet: ceph-common-2:16.2.4-0.el8.x86_64 24/29 2023-11-17T05:15:32.375 INFO:teuthology.orchestra.run.smithi066.stdout: Installing : ceph-base-2:16.2.4-0.el8.x86_64 25/29 2023-11-17T05:15:32.511 INFO:teuthology.orchestra.run.smithi066.stdout: Running scriptlet: ceph-base-2:16.2.4-0.el8.x86_64 25/29 2023-11-17T05:15:32.637 INFO:teuthology.orchestra.run.smithi066.stdout: Installing : ceph-selinux-2:16.2.4-0.el8.x86_64 26/29 2023-11-17T05:15:32.909 INFO:teuthology.orchestra.run.smithi040.stdout: Preparing : 1/1 2023-11-17T05:15:33.135 INFO:teuthology.orchestra.run.smithi040.stdout: Installing : fmt-6.2.1-1.el8.x86_64 1/29 2023-11-17T05:15:33.344 INFO:teuthology.orchestra.run.smithi040.stdout: Installing : liboath-2.6.2-3.el8.x86_64 2/29 2023-11-17T05:15:33.712 INFO:teuthology.orchestra.run.smithi040.stdout: Installing : lttng-ust-2.8.1-11.el8.x86_64 3/29 2023-11-17T05:15:34.152 INFO:teuthology.orchestra.run.smithi040.stdout: Running scriptlet: lttng-ust-2.8.1-11.el8.x86_64 3/29 2023-11-17T05:15:34.333 INFO:teuthology.orchestra.run.smithi040.stdout: Upgrading : librados2-2:16.2.4-0.el8.x86_64 4/29 2023-11-17T05:15:34.443 INFO:teuthology.orchestra.run.smithi040.stdout: Running scriptlet: librados2-2:16.2.4-0.el8.x86_64 4/29 2023-11-17T05:15:34.682 INFO:teuthology.orchestra.run.smithi040.stdout: Installing : python3-rados-2:16.2.4-0.el8.x86_64 5/29 2023-11-17T05:15:34.825 INFO:teuthology.orchestra.run.smithi040.stdout: Installing : libcephfs2-2:16.2.4-0.el8.x86_64 6/29 2023-11-17T05:15:34.878 INFO:teuthology.orchestra.run.smithi040.stdout: Running scriptlet: libcephfs2-2:16.2.4-0.el8.x86_64 6/29 2023-11-17T05:15:35.085 INFO:teuthology.orchestra.run.smithi040.stdout: Installing : librabbitmq-0.9.0-4.el8.x86_64 7/29 2023-11-17T05:15:35.258 INFO:teuthology.orchestra.run.smithi040.stdout: Installing : librdkafka-0.11.4-3.el8.x86_64 8/29 2023-11-17T05:15:35.723 INFO:teuthology.orchestra.run.smithi040.stdout: Running scriptlet: librdkafka-0.11.4-3.el8.x86_64 8/29 2023-11-17T05:15:35.867 INFO:teuthology.orchestra.run.smithi040.stdout: Installing : librgw2-2:16.2.4-0.el8.x86_64 9/29 2023-11-17T05:15:35.952 INFO:teuthology.orchestra.run.smithi040.stdout: Running scriptlet: librgw2-2:16.2.4-0.el8.x86_64 9/29 2023-11-17T05:15:36.122 INFO:teuthology.orchestra.run.smithi040.stdout: Installing : leveldb-1.22-1.el8.x86_64 10/29 2023-11-17T05:15:36.332 INFO:teuthology.orchestra.run.smithi040.stdout: Installing : daxctl-libs-71.1-7.el8.x86_64 11/29 2023-11-17T05:15:36.384 INFO:teuthology.orchestra.run.smithi040.stdout: Running scriptlet: daxctl-libs-71.1-7.el8.x86_64 11/29 2023-11-17T05:15:36.550 INFO:teuthology.orchestra.run.smithi040.stdout: Installing : python3-ceph-argparse-2:16.2.4-0.el8.x86_64 12/29 2023-11-17T05:15:36.710 INFO:teuthology.orchestra.run.smithi040.stdout: Installing : python3-cephfs-2:16.2.4-0.el8.x86_64 13/29 2023-11-17T05:15:36.891 INFO:teuthology.orchestra.run.smithi040.stdout: Installing : ndctl-libs-71.1-7.el8.x86_64 14/29 2023-11-17T05:15:36.969 INFO:teuthology.orchestra.run.smithi040.stdout: Running scriptlet: ndctl-libs-71.1-7.el8.x86_64 14/29 2023-11-17T05:15:37.133 INFO:teuthology.orchestra.run.smithi040.stdout: Installing : libpmemobj-1.6.1-1.el8.x86_64 15/29 2023-11-17T05:15:37.611 INFO:teuthology.orchestra.run.smithi040.stdout: Running scriptlet: libpmemobj-1.6.1-1.el8.x86_64 15/29 2023-11-17T05:15:37.791 INFO:teuthology.orchestra.run.smithi040.stdout: Upgrading : librbd1-2:16.2.4-0.el8.x86_64 16/29 2023-11-17T05:15:37.905 INFO:teuthology.orchestra.run.smithi040.stdout: Running scriptlet: librbd1-2:16.2.4-0.el8.x86_64 16/29 2023-11-17T05:15:38.080 INFO:teuthology.orchestra.run.smithi040.stdout: Installing : python3-rbd-2:16.2.4-0.el8.x86_64 17/29 2023-11-17T05:15:38.294 INFO:teuthology.orchestra.run.smithi040.stdout: Installing : python3-rgw-2:16.2.4-0.el8.x86_64 18/29 2023-11-17T05:15:38.449 INFO:teuthology.orchestra.run.smithi040.stdout: Installing : libradosstriper1-2:16.2.4-0.el8.x86_64 19/29 2023-11-17T05:15:38.538 INFO:teuthology.orchestra.run.smithi040.stdout: Running scriptlet: libradosstriper1-2:16.2.4-0.el8.x86_64 19/29 2023-11-17T05:15:38.755 INFO:teuthology.orchestra.run.smithi040.stdout: Installing : libunwind-1.3.1-3.el8.x86_64 20/29 2023-11-17T05:15:38.942 INFO:teuthology.orchestra.run.smithi040.stdout: Installing : gperftools-libs-1:2.7-9.el8.x86_64 21/29 2023-11-17T05:15:39.131 INFO:teuthology.orchestra.run.smithi040.stdout: Installing : python3-prettytable-0.7.2-14.el8.noarch 22/29 2023-11-17T05:15:39.358 INFO:teuthology.orchestra.run.smithi040.stdout: Installing : python3-ceph-common-2:16.2.4-0.el8.x86_64 23/29 2023-11-17T05:15:42.206 INFO:teuthology.orchestra.run.smithi040.stdout: Running scriptlet: ceph-common-2:16.2.4-0.el8.x86_64 24/29 2023-11-17T05:15:42.416 INFO:teuthology.orchestra.run.smithi040.stdout: Installing : ceph-common-2:16.2.4-0.el8.x86_64 24/29 2023-11-17T05:15:43.234 INFO:teuthology.orchestra.run.smithi040.stdout: Running scriptlet: ceph-common-2:16.2.4-0.el8.x86_64 24/29 2023-11-17T05:15:43.527 INFO:teuthology.orchestra.run.smithi040.stdout: Installing : ceph-base-2:16.2.4-0.el8.x86_64 25/29 2023-11-17T05:15:43.652 INFO:teuthology.orchestra.run.smithi040.stdout: Running scriptlet: ceph-base-2:16.2.4-0.el8.x86_64 25/29 2023-11-17T05:15:43.782 INFO:teuthology.orchestra.run.smithi040.stdout: Installing : ceph-selinux-2:16.2.4-0.el8.x86_64 26/29 2023-11-17T05:15:52.117 INFO:teuthology.orchestra.run.smithi066.stdout: Running scriptlet: ceph-selinux-2:16.2.4-0.el8.x86_64 26/29 2023-11-17T05:15:52.117 INFO:teuthology.orchestra.run.smithi066.stdout:skipping the directory /sys 2023-11-17T05:15:52.117 INFO:teuthology.orchestra.run.smithi066.stdout:skipping the directory /proc 2023-11-17T05:15:52.117 INFO:teuthology.orchestra.run.smithi066.stdout:skipping the directory /dev 2023-11-17T05:15:52.118 INFO:teuthology.orchestra.run.smithi066.stdout:skipping the directory /run 2023-11-17T05:15:52.118 INFO:teuthology.orchestra.run.smithi066.stdout:skipping the directory /mnt 2023-11-17T05:15:52.118 INFO:teuthology.orchestra.run.smithi066.stdout:skipping the directory /var/tmp 2023-11-17T05:15:52.118 INFO:teuthology.orchestra.run.smithi066.stdout:skipping the directory /home 2023-11-17T05:15:52.118 INFO:teuthology.orchestra.run.smithi066.stdout:skipping the directory /tmp 2023-11-17T05:15:52.118 INFO:teuthology.orchestra.run.smithi066.stdout:skipping the directory /dev 2023-11-17T05:15:52.118 INFO:teuthology.orchestra.run.smithi066.stdout: 2023-11-17T05:15:53.556 INFO:teuthology.orchestra.run.smithi066.stdout: Installing : ceph-radosgw-2:16.2.4-0.el8.x86_64 27/29 2023-11-17T05:15:53.634 INFO:teuthology.orchestra.run.smithi066.stdout: Running scriptlet: ceph-radosgw-2:16.2.4-0.el8.x86_64 27/29 2023-11-17T05:15:53.634 INFO:teuthology.orchestra.run.smithi066.stdout: Cleanup : librbd1-1:12.2.7-9.el8.x86_64 28/29 2023-11-17T05:15:53.829 INFO:teuthology.orchestra.run.smithi066.stdout: Running scriptlet: librbd1-1:12.2.7-9.el8.x86_64 28/29 2023-11-17T05:15:53.829 INFO:teuthology.orchestra.run.smithi066.stdout: Cleanup : librados2-1:12.2.7-9.el8.x86_64 29/29 2023-11-17T05:15:54.565 INFO:teuthology.orchestra.run.smithi066.stdout: Running scriptlet: librados2-1:12.2.7-9.el8.x86_64 29/29 2023-11-17T05:15:54.565 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : ceph-base-2:16.2.4-0.el8.x86_64 1/29 2023-11-17T05:15:54.565 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : ceph-common-2:16.2.4-0.el8.x86_64 2/29 2023-11-17T05:15:54.565 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : ceph-radosgw-2:16.2.4-0.el8.x86_64 3/29 2023-11-17T05:15:54.565 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : ceph-selinux-2:16.2.4-0.el8.x86_64 4/29 2023-11-17T05:15:54.565 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : libcephfs2-2:16.2.4-0.el8.x86_64 5/29 2023-11-17T05:15:54.565 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : libradosstriper1-2:16.2.4-0.el8.x86_64 6/29 2023-11-17T05:15:54.565 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : librgw2-2:16.2.4-0.el8.x86_64 7/29 2023-11-17T05:15:54.566 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : python3-ceph-argparse-2:16.2.4-0.el8.x86_64 8/29 2023-11-17T05:15:54.566 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : python3-ceph-common-2:16.2.4-0.el8.x86_64 9/29 2023-11-17T05:15:54.566 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : python3-cephfs-2:16.2.4-0.el8.x86_64 10/29 2023-11-17T05:15:54.566 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : python3-rados-2:16.2.4-0.el8.x86_64 11/29 2023-11-17T05:15:54.566 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : python3-rbd-2:16.2.4-0.el8.x86_64 12/29 2023-11-17T05:15:54.566 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : python3-rgw-2:16.2.4-0.el8.x86_64 13/29 2023-11-17T05:15:54.566 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : libpmemobj-1.6.1-1.el8.x86_64 14/29 2023-11-17T05:15:54.566 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : librdkafka-0.11.4-3.el8.x86_64 15/29 2023-11-17T05:15:54.566 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : lttng-ust-2.8.1-11.el8.x86_64 16/29 2023-11-17T05:15:54.566 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : python3-prettytable-0.7.2-14.el8.noarch 17/29 2023-11-17T05:15:54.566 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : daxctl-libs-71.1-7.el8.x86_64 18/29 2023-11-17T05:15:54.567 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : librabbitmq-0.9.0-4.el8.x86_64 19/29 2023-11-17T05:15:54.567 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : ndctl-libs-71.1-7.el8.x86_64 20/29 2023-11-17T05:15:54.567 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : fmt-6.2.1-1.el8.x86_64 21/29 2023-11-17T05:15:54.567 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : gperftools-libs-1:2.7-9.el8.x86_64 22/29 2023-11-17T05:15:54.567 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : leveldb-1.22-1.el8.x86_64 23/29 2023-11-17T05:15:54.567 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : liboath-2.6.2-3.el8.x86_64 24/29 2023-11-17T05:15:54.567 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : libunwind-1.3.1-3.el8.x86_64 25/29 2023-11-17T05:15:54.567 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : librados2-2:16.2.4-0.el8.x86_64 26/29 2023-11-17T05:15:54.567 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : librados2-1:12.2.7-9.el8.x86_64 27/29 2023-11-17T05:15:54.567 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : librbd1-2:16.2.4-0.el8.x86_64 28/29 2023-11-17T05:15:55.862 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : librbd1-1:12.2.7-9.el8.x86_64 29/29 2023-11-17T05:15:55.863 INFO:teuthology.orchestra.run.smithi066.stdout: 2023-11-17T05:15:55.863 INFO:teuthology.orchestra.run.smithi066.stdout:Upgraded: 2023-11-17T05:15:55.863 INFO:teuthology.orchestra.run.smithi066.stdout: librados2-2:16.2.4-0.el8.x86_64 librbd1-2:16.2.4-0.el8.x86_64 2023-11-17T05:15:55.863 INFO:teuthology.orchestra.run.smithi066.stdout:Installed: 2023-11-17T05:15:55.863 INFO:teuthology.orchestra.run.smithi066.stdout: ceph-base-2:16.2.4-0.el8.x86_64 2023-11-17T05:15:55.863 INFO:teuthology.orchestra.run.smithi066.stdout: ceph-common-2:16.2.4-0.el8.x86_64 2023-11-17T05:15:55.863 INFO:teuthology.orchestra.run.smithi066.stdout: ceph-radosgw-2:16.2.4-0.el8.x86_64 2023-11-17T05:15:55.863 INFO:teuthology.orchestra.run.smithi066.stdout: ceph-selinux-2:16.2.4-0.el8.x86_64 2023-11-17T05:15:55.863 INFO:teuthology.orchestra.run.smithi066.stdout: daxctl-libs-71.1-7.el8.x86_64 2023-11-17T05:15:55.864 INFO:teuthology.orchestra.run.smithi066.stdout: fmt-6.2.1-1.el8.x86_64 2023-11-17T05:15:55.864 INFO:teuthology.orchestra.run.smithi066.stdout: gperftools-libs-1:2.7-9.el8.x86_64 2023-11-17T05:15:55.864 INFO:teuthology.orchestra.run.smithi066.stdout: leveldb-1.22-1.el8.x86_64 2023-11-17T05:15:55.864 INFO:teuthology.orchestra.run.smithi066.stdout: libcephfs2-2:16.2.4-0.el8.x86_64 2023-11-17T05:15:55.864 INFO:teuthology.orchestra.run.smithi066.stdout: liboath-2.6.2-3.el8.x86_64 2023-11-17T05:15:55.864 INFO:teuthology.orchestra.run.smithi066.stdout: libpmemobj-1.6.1-1.el8.x86_64 2023-11-17T05:15:55.864 INFO:teuthology.orchestra.run.smithi066.stdout: librabbitmq-0.9.0-4.el8.x86_64 2023-11-17T05:15:55.864 INFO:teuthology.orchestra.run.smithi066.stdout: libradosstriper1-2:16.2.4-0.el8.x86_64 2023-11-17T05:15:55.864 INFO:teuthology.orchestra.run.smithi066.stdout: librdkafka-0.11.4-3.el8.x86_64 2023-11-17T05:15:55.864 INFO:teuthology.orchestra.run.smithi066.stdout: librgw2-2:16.2.4-0.el8.x86_64 2023-11-17T05:15:55.864 INFO:teuthology.orchestra.run.smithi066.stdout: libunwind-1.3.1-3.el8.x86_64 2023-11-17T05:15:55.864 INFO:teuthology.orchestra.run.smithi066.stdout: lttng-ust-2.8.1-11.el8.x86_64 2023-11-17T05:15:55.865 INFO:teuthology.orchestra.run.smithi066.stdout: ndctl-libs-71.1-7.el8.x86_64 2023-11-17T05:15:55.865 INFO:teuthology.orchestra.run.smithi066.stdout: python3-ceph-argparse-2:16.2.4-0.el8.x86_64 2023-11-17T05:15:55.865 INFO:teuthology.orchestra.run.smithi066.stdout: python3-ceph-common-2:16.2.4-0.el8.x86_64 2023-11-17T05:15:55.865 INFO:teuthology.orchestra.run.smithi066.stdout: python3-cephfs-2:16.2.4-0.el8.x86_64 2023-11-17T05:15:55.865 INFO:teuthology.orchestra.run.smithi066.stdout: python3-prettytable-0.7.2-14.el8.noarch 2023-11-17T05:15:55.865 INFO:teuthology.orchestra.run.smithi066.stdout: python3-rados-2:16.2.4-0.el8.x86_64 2023-11-17T05:15:55.865 INFO:teuthology.orchestra.run.smithi066.stdout: python3-rbd-2:16.2.4-0.el8.x86_64 2023-11-17T05:15:55.865 INFO:teuthology.orchestra.run.smithi066.stdout: python3-rgw-2:16.2.4-0.el8.x86_64 2023-11-17T05:15:55.865 INFO:teuthology.orchestra.run.smithi066.stdout: 2023-11-17T05:15:55.865 INFO:teuthology.orchestra.run.smithi066.stdout:Complete! 2023-11-17T05:15:56.047 DEBUG:teuthology.orchestra.run.smithi066:> sudo yum -y install ceph-test 2023-11-17T05:15:56.935 INFO:teuthology.orchestra.run.smithi066.stdout:Last metadata expiration check: 0:00:43 ago on Fri 17 Nov 2023 05:15:13 AM UTC. 2023-11-17T05:15:58.708 INFO:teuthology.orchestra.run.smithi066.stdout:Dependencies resolved. 2023-11-17T05:15:58.709 INFO:teuthology.orchestra.run.smithi066.stdout:================================================================================ 2023-11-17T05:15:58.709 INFO:teuthology.orchestra.run.smithi066.stdout: Package Architecture Version Repository Size 2023-11-17T05:15:58.709 INFO:teuthology.orchestra.run.smithi066.stdout:================================================================================ 2023-11-17T05:15:58.709 INFO:teuthology.orchestra.run.smithi066.stdout:Installing: 2023-11-17T05:15:58.709 INFO:teuthology.orchestra.run.smithi066.stdout: ceph-test x86_64 2:16.2.4-0.el8 ceph 52 M 2023-11-17T05:15:58.709 INFO:teuthology.orchestra.run.smithi066.stdout:Installing dependencies: 2023-11-17T05:15:58.709 INFO:teuthology.orchestra.run.smithi066.stdout: jq x86_64 1.6-8.el8 appstream 203 k 2023-11-17T05:15:58.710 INFO:teuthology.orchestra.run.smithi066.stdout: libcephsqlite x86_64 2:16.2.4-0.el8 ceph 169 k 2023-11-17T05:15:58.710 INFO:teuthology.orchestra.run.smithi066.stdout: oniguruma x86_64 6.8.2-2.el8 appstream 187 k 2023-11-17T05:15:58.710 INFO:teuthology.orchestra.run.smithi066.stdout: socat x86_64 1.7.4.1-1.el8 appstream 323 k 2023-11-17T05:15:58.710 INFO:teuthology.orchestra.run.smithi066.stdout: xmlstarlet x86_64 1.6.1-20.el8 appstream 71 k 2023-11-17T05:15:58.710 INFO:teuthology.orchestra.run.smithi066.stdout: 2023-11-17T05:15:58.710 INFO:teuthology.orchestra.run.smithi066.stdout:Transaction Summary 2023-11-17T05:15:58.710 INFO:teuthology.orchestra.run.smithi066.stdout:================================================================================ 2023-11-17T05:15:58.710 INFO:teuthology.orchestra.run.smithi066.stdout:Install 6 Packages 2023-11-17T05:15:58.710 INFO:teuthology.orchestra.run.smithi066.stdout: 2023-11-17T05:15:58.711 INFO:teuthology.orchestra.run.smithi066.stdout:Total download size: 53 M 2023-11-17T05:15:58.711 INFO:teuthology.orchestra.run.smithi066.stdout:Installed size: 229 M 2023-11-17T05:15:58.711 INFO:teuthology.orchestra.run.smithi066.stdout:Downloading Packages: 2023-11-17T05:15:58.816 INFO:teuthology.orchestra.run.smithi066.stdout:(1/6): libcephsqlite-16.2.4-0.el8.x86_64.rpm 2.5 MB/s | 169 kB 00:00 2023-11-17T05:15:58.966 INFO:teuthology.orchestra.run.smithi066.stdout:(2/6): jq-1.6-8.el8.x86_64.rpm 940 kB/s | 203 kB 00:00 2023-11-17T05:15:59.033 INFO:teuthology.orchestra.run.smithi066.stdout:(3/6): oniguruma-6.8.2-2.el8.x86_64.rpm 864 kB/s | 187 kB 00:00 2023-11-17T05:15:59.550 INFO:teuthology.orchestra.run.smithi066.stdout:(4/6): ceph-test-16.2.4-0.el8.x86_64.rpm 65 MB/s | 52 MB 00:00 2023-11-17T05:15:59.576 INFO:teuthology.orchestra.run.smithi066.stdout:(5/6): socat-1.7.4.1-1.el8.x86_64.rpm 530 kB/s | 323 kB 00:00 2023-11-17T05:15:59.601 INFO:teuthology.orchestra.run.smithi066.stdout:(6/6): xmlstarlet-1.6.1-20.el8.x86_64.rpm 124 kB/s | 71 kB 00:00 2023-11-17T05:15:59.603 INFO:teuthology.orchestra.run.smithi066.stdout:-------------------------------------------------------------------------------- 2023-11-17T05:15:59.603 INFO:teuthology.orchestra.run.smithi066.stdout:Total 59 MB/s | 53 MB 00:00 2023-11-17T05:15:59.648 INFO:teuthology.orchestra.run.smithi066.stdout:Running transaction check 2023-11-17T05:15:59.671 INFO:teuthology.orchestra.run.smithi066.stdout:Transaction check succeeded. 2023-11-17T05:15:59.671 INFO:teuthology.orchestra.run.smithi066.stdout:Running transaction test 2023-11-17T05:15:59.995 INFO:teuthology.orchestra.run.smithi066.stdout:Transaction test succeeded. 2023-11-17T05:15:59.998 INFO:teuthology.orchestra.run.smithi066.stdout:Running transaction 2023-11-17T05:16:00.485 INFO:teuthology.orchestra.run.smithi066.stdout: Preparing : 1/1 2023-11-17T05:16:00.714 INFO:teuthology.orchestra.run.smithi066.stdout: Installing : xmlstarlet-1.6.1-20.el8.x86_64 1/6 2023-11-17T05:16:00.879 INFO:teuthology.orchestra.run.smithi066.stdout: Installing : socat-1.7.4.1-1.el8.x86_64 2/6 2023-11-17T05:16:01.075 INFO:teuthology.orchestra.run.smithi066.stdout: Installing : oniguruma-6.8.2-2.el8.x86_64 3/6 2023-11-17T05:16:01.158 INFO:teuthology.orchestra.run.smithi066.stdout: Running scriptlet: oniguruma-6.8.2-2.el8.x86_64 3/6 2023-11-17T05:16:01.333 INFO:teuthology.orchestra.run.smithi066.stdout: Installing : jq-1.6-8.el8.x86_64 4/6 2023-11-17T05:16:01.500 INFO:teuthology.orchestra.run.smithi066.stdout: Installing : libcephsqlite-2:16.2.4-0.el8.x86_64 5/6 2023-11-17T05:16:05.042 INFO:teuthology.orchestra.run.smithi040.stdout: Running scriptlet: ceph-selinux-2:16.2.4-0.el8.x86_64 26/29 2023-11-17T05:16:05.042 INFO:teuthology.orchestra.run.smithi040.stdout:skipping the directory /sys 2023-11-17T05:16:05.042 INFO:teuthology.orchestra.run.smithi040.stdout:skipping the directory /proc 2023-11-17T05:16:05.042 INFO:teuthology.orchestra.run.smithi040.stdout:skipping the directory /dev 2023-11-17T05:16:05.042 INFO:teuthology.orchestra.run.smithi040.stdout:skipping the directory /run 2023-11-17T05:16:05.043 INFO:teuthology.orchestra.run.smithi040.stdout:skipping the directory /mnt 2023-11-17T05:16:05.043 INFO:teuthology.orchestra.run.smithi040.stdout:skipping the directory /var/tmp 2023-11-17T05:16:05.043 INFO:teuthology.orchestra.run.smithi040.stdout:skipping the directory /home 2023-11-17T05:16:05.043 INFO:teuthology.orchestra.run.smithi040.stdout:skipping the directory /tmp 2023-11-17T05:16:05.043 INFO:teuthology.orchestra.run.smithi040.stdout:skipping the directory /dev 2023-11-17T05:16:05.043 INFO:teuthology.orchestra.run.smithi040.stdout: 2023-11-17T05:16:06.490 INFO:teuthology.orchestra.run.smithi040.stdout: Installing : ceph-radosgw-2:16.2.4-0.el8.x86_64 27/29 2023-11-17T05:16:06.657 INFO:teuthology.orchestra.run.smithi066.stdout: Running scriptlet: libcephsqlite-2:16.2.4-0.el8.x86_64 5/6 2023-11-17T05:16:06.703 INFO:teuthology.orchestra.run.smithi040.stdout: Running scriptlet: ceph-radosgw-2:16.2.4-0.el8.x86_64 27/29 2023-11-17T05:16:06.703 INFO:teuthology.orchestra.run.smithi040.stdout: Cleanup : librbd1-1:12.2.7-9.el8.x86_64 28/29 2023-11-17T05:16:06.890 INFO:teuthology.orchestra.run.smithi066.stdout: Installing : ceph-test-2:16.2.4-0.el8.x86_64 6/6 2023-11-17T05:16:06.898 INFO:teuthology.orchestra.run.smithi040.stdout: Running scriptlet: librbd1-1:12.2.7-9.el8.x86_64 28/29 2023-11-17T05:16:06.899 INFO:teuthology.orchestra.run.smithi040.stdout: Cleanup : librados2-1:12.2.7-9.el8.x86_64 29/29 2023-11-17T05:16:07.199 INFO:teuthology.orchestra.run.smithi066.stdout: Running scriptlet: ceph-test-2:16.2.4-0.el8.x86_64 6/6 2023-11-17T05:16:07.199 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : ceph-test-2:16.2.4-0.el8.x86_64 1/6 2023-11-17T05:16:07.199 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : libcephsqlite-2:16.2.4-0.el8.x86_64 2/6 2023-11-17T05:16:07.199 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : jq-1.6-8.el8.x86_64 3/6 2023-11-17T05:16:07.199 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : oniguruma-6.8.2-2.el8.x86_64 4/6 2023-11-17T05:16:07.199 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : socat-1.7.4.1-1.el8.x86_64 5/6 2023-11-17T05:16:07.648 INFO:teuthology.orchestra.run.smithi040.stdout: Running scriptlet: librados2-1:12.2.7-9.el8.x86_64 29/29 2023-11-17T05:16:07.648 INFO:teuthology.orchestra.run.smithi040.stdout: Verifying : ceph-base-2:16.2.4-0.el8.x86_64 1/29 2023-11-17T05:16:07.648 INFO:teuthology.orchestra.run.smithi040.stdout: Verifying : ceph-common-2:16.2.4-0.el8.x86_64 2/29 2023-11-17T05:16:07.649 INFO:teuthology.orchestra.run.smithi040.stdout: Verifying : ceph-radosgw-2:16.2.4-0.el8.x86_64 3/29 2023-11-17T05:16:07.649 INFO:teuthology.orchestra.run.smithi040.stdout: Verifying : ceph-selinux-2:16.2.4-0.el8.x86_64 4/29 2023-11-17T05:16:07.649 INFO:teuthology.orchestra.run.smithi040.stdout: Verifying : libcephfs2-2:16.2.4-0.el8.x86_64 5/29 2023-11-17T05:16:07.649 INFO:teuthology.orchestra.run.smithi040.stdout: Verifying : libradosstriper1-2:16.2.4-0.el8.x86_64 6/29 2023-11-17T05:16:07.649 INFO:teuthology.orchestra.run.smithi040.stdout: Verifying : librgw2-2:16.2.4-0.el8.x86_64 7/29 2023-11-17T05:16:07.649 INFO:teuthology.orchestra.run.smithi040.stdout: Verifying : python3-ceph-argparse-2:16.2.4-0.el8.x86_64 8/29 2023-11-17T05:16:07.649 INFO:teuthology.orchestra.run.smithi040.stdout: Verifying : python3-ceph-common-2:16.2.4-0.el8.x86_64 9/29 2023-11-17T05:16:07.649 INFO:teuthology.orchestra.run.smithi040.stdout: Verifying : python3-cephfs-2:16.2.4-0.el8.x86_64 10/29 2023-11-17T05:16:07.650 INFO:teuthology.orchestra.run.smithi040.stdout: Verifying : python3-rados-2:16.2.4-0.el8.x86_64 11/29 2023-11-17T05:16:07.650 INFO:teuthology.orchestra.run.smithi040.stdout: Verifying : python3-rbd-2:16.2.4-0.el8.x86_64 12/29 2023-11-17T05:16:07.650 INFO:teuthology.orchestra.run.smithi040.stdout: Verifying : python3-rgw-2:16.2.4-0.el8.x86_64 13/29 2023-11-17T05:16:07.650 INFO:teuthology.orchestra.run.smithi040.stdout: Verifying : libpmemobj-1.6.1-1.el8.x86_64 14/29 2023-11-17T05:16:07.650 INFO:teuthology.orchestra.run.smithi040.stdout: Verifying : librdkafka-0.11.4-3.el8.x86_64 15/29 2023-11-17T05:16:07.650 INFO:teuthology.orchestra.run.smithi040.stdout: Verifying : lttng-ust-2.8.1-11.el8.x86_64 16/29 2023-11-17T05:16:07.650 INFO:teuthology.orchestra.run.smithi040.stdout: Verifying : python3-prettytable-0.7.2-14.el8.noarch 17/29 2023-11-17T05:16:07.650 INFO:teuthology.orchestra.run.smithi040.stdout: Verifying : daxctl-libs-71.1-7.el8.x86_64 18/29 2023-11-17T05:16:07.650 INFO:teuthology.orchestra.run.smithi040.stdout: Verifying : librabbitmq-0.9.0-4.el8.x86_64 19/29 2023-11-17T05:16:07.650 INFO:teuthology.orchestra.run.smithi040.stdout: Verifying : ndctl-libs-71.1-7.el8.x86_64 20/29 2023-11-17T05:16:07.650 INFO:teuthology.orchestra.run.smithi040.stdout: Verifying : fmt-6.2.1-1.el8.x86_64 21/29 2023-11-17T05:16:07.651 INFO:teuthology.orchestra.run.smithi040.stdout: Verifying : gperftools-libs-1:2.7-9.el8.x86_64 22/29 2023-11-17T05:16:07.651 INFO:teuthology.orchestra.run.smithi040.stdout: Verifying : leveldb-1.22-1.el8.x86_64 23/29 2023-11-17T05:16:07.651 INFO:teuthology.orchestra.run.smithi040.stdout: Verifying : liboath-2.6.2-3.el8.x86_64 24/29 2023-11-17T05:16:07.651 INFO:teuthology.orchestra.run.smithi040.stdout: Verifying : libunwind-1.3.1-3.el8.x86_64 25/29 2023-11-17T05:16:07.651 INFO:teuthology.orchestra.run.smithi040.stdout: Verifying : librados2-2:16.2.4-0.el8.x86_64 26/29 2023-11-17T05:16:07.651 INFO:teuthology.orchestra.run.smithi040.stdout: Verifying : librados2-1:12.2.7-9.el8.x86_64 27/29 2023-11-17T05:16:07.652 INFO:teuthology.orchestra.run.smithi040.stdout: Verifying : librbd1-2:16.2.4-0.el8.x86_64 28/29 2023-11-17T05:16:07.924 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : xmlstarlet-1.6.1-20.el8.x86_64 6/6 2023-11-17T05:16:07.924 INFO:teuthology.orchestra.run.smithi066.stdout: 2023-11-17T05:16:07.924 INFO:teuthology.orchestra.run.smithi066.stdout:Installed: 2023-11-17T05:16:07.924 INFO:teuthology.orchestra.run.smithi066.stdout: ceph-test-2:16.2.4-0.el8.x86_64 jq-1.6-8.el8.x86_64 2023-11-17T05:16:07.924 INFO:teuthology.orchestra.run.smithi066.stdout: libcephsqlite-2:16.2.4-0.el8.x86_64 oniguruma-6.8.2-2.el8.x86_64 2023-11-17T05:16:07.924 INFO:teuthology.orchestra.run.smithi066.stdout: socat-1.7.4.1-1.el8.x86_64 xmlstarlet-1.6.1-20.el8.x86_64 2023-11-17T05:16:07.924 INFO:teuthology.orchestra.run.smithi066.stdout: 2023-11-17T05:16:07.924 INFO:teuthology.orchestra.run.smithi066.stdout:Complete! 2023-11-17T05:16:08.157 DEBUG:teuthology.orchestra.run.smithi066:> sudo yum -y install ceph 2023-11-17T05:16:08.831 INFO:teuthology.orchestra.run.smithi040.stdout: Verifying : librbd1-1:12.2.7-9.el8.x86_64 29/29 2023-11-17T05:16:08.831 INFO:teuthology.orchestra.run.smithi040.stdout: 2023-11-17T05:16:08.831 INFO:teuthology.orchestra.run.smithi040.stdout:Upgraded: 2023-11-17T05:16:08.831 INFO:teuthology.orchestra.run.smithi040.stdout: librados2-2:16.2.4-0.el8.x86_64 librbd1-2:16.2.4-0.el8.x86_64 2023-11-17T05:16:08.831 INFO:teuthology.orchestra.run.smithi040.stdout:Installed: 2023-11-17T05:16:08.831 INFO:teuthology.orchestra.run.smithi040.stdout: ceph-base-2:16.2.4-0.el8.x86_64 2023-11-17T05:16:08.831 INFO:teuthology.orchestra.run.smithi040.stdout: ceph-common-2:16.2.4-0.el8.x86_64 2023-11-17T05:16:08.831 INFO:teuthology.orchestra.run.smithi040.stdout: ceph-radosgw-2:16.2.4-0.el8.x86_64 2023-11-17T05:16:08.831 INFO:teuthology.orchestra.run.smithi040.stdout: ceph-selinux-2:16.2.4-0.el8.x86_64 2023-11-17T05:16:08.831 INFO:teuthology.orchestra.run.smithi040.stdout: daxctl-libs-71.1-7.el8.x86_64 2023-11-17T05:16:08.831 INFO:teuthology.orchestra.run.smithi040.stdout: fmt-6.2.1-1.el8.x86_64 2023-11-17T05:16:08.832 INFO:teuthology.orchestra.run.smithi040.stdout: gperftools-libs-1:2.7-9.el8.x86_64 2023-11-17T05:16:08.832 INFO:teuthology.orchestra.run.smithi040.stdout: leveldb-1.22-1.el8.x86_64 2023-11-17T05:16:08.832 INFO:teuthology.orchestra.run.smithi040.stdout: libcephfs2-2:16.2.4-0.el8.x86_64 2023-11-17T05:16:08.832 INFO:teuthology.orchestra.run.smithi040.stdout: liboath-2.6.2-3.el8.x86_64 2023-11-17T05:16:08.832 INFO:teuthology.orchestra.run.smithi040.stdout: libpmemobj-1.6.1-1.el8.x86_64 2023-11-17T05:16:08.832 INFO:teuthology.orchestra.run.smithi040.stdout: librabbitmq-0.9.0-4.el8.x86_64 2023-11-17T05:16:08.832 INFO:teuthology.orchestra.run.smithi040.stdout: libradosstriper1-2:16.2.4-0.el8.x86_64 2023-11-17T05:16:08.832 INFO:teuthology.orchestra.run.smithi040.stdout: librdkafka-0.11.4-3.el8.x86_64 2023-11-17T05:16:08.832 INFO:teuthology.orchestra.run.smithi040.stdout: librgw2-2:16.2.4-0.el8.x86_64 2023-11-17T05:16:08.832 INFO:teuthology.orchestra.run.smithi040.stdout: libunwind-1.3.1-3.el8.x86_64 2023-11-17T05:16:08.832 INFO:teuthology.orchestra.run.smithi040.stdout: lttng-ust-2.8.1-11.el8.x86_64 2023-11-17T05:16:08.832 INFO:teuthology.orchestra.run.smithi040.stdout: ndctl-libs-71.1-7.el8.x86_64 2023-11-17T05:16:08.833 INFO:teuthology.orchestra.run.smithi040.stdout: python3-ceph-argparse-2:16.2.4-0.el8.x86_64 2023-11-17T05:16:08.833 INFO:teuthology.orchestra.run.smithi040.stdout: python3-ceph-common-2:16.2.4-0.el8.x86_64 2023-11-17T05:16:08.833 INFO:teuthology.orchestra.run.smithi040.stdout: python3-cephfs-2:16.2.4-0.el8.x86_64 2023-11-17T05:16:08.833 INFO:teuthology.orchestra.run.smithi040.stdout: python3-prettytable-0.7.2-14.el8.noarch 2023-11-17T05:16:08.833 INFO:teuthology.orchestra.run.smithi040.stdout: python3-rados-2:16.2.4-0.el8.x86_64 2023-11-17T05:16:08.833 INFO:teuthology.orchestra.run.smithi040.stdout: python3-rbd-2:16.2.4-0.el8.x86_64 2023-11-17T05:16:08.833 INFO:teuthology.orchestra.run.smithi040.stdout: python3-rgw-2:16.2.4-0.el8.x86_64 2023-11-17T05:16:08.833 INFO:teuthology.orchestra.run.smithi040.stdout: 2023-11-17T05:16:08.833 INFO:teuthology.orchestra.run.smithi040.stdout:Complete! 2023-11-17T05:16:09.030 DEBUG:teuthology.orchestra.run.smithi040:> sudo yum -y install ceph-test 2023-11-17T05:16:09.044 INFO:teuthology.orchestra.run.smithi066.stdout:Last metadata expiration check: 0:00:56 ago on Fri 17 Nov 2023 05:15:13 AM UTC. 2023-11-17T05:16:10.028 INFO:teuthology.orchestra.run.smithi040.stdout:Last metadata expiration check: 0:00:52 ago on Fri 17 Nov 2023 05:15:18 AM UTC. 2023-11-17T05:16:10.859 INFO:teuthology.orchestra.run.smithi066.stdout:Dependencies resolved. 2023-11-17T05:16:10.862 INFO:teuthology.orchestra.run.smithi066.stdout:================================================================================ 2023-11-17T05:16:10.862 INFO:teuthology.orchestra.run.smithi066.stdout: Package Arch Version Repository Size 2023-11-17T05:16:10.862 INFO:teuthology.orchestra.run.smithi066.stdout:================================================================================ 2023-11-17T05:16:10.863 INFO:teuthology.orchestra.run.smithi066.stdout:Installing: 2023-11-17T05:16:10.863 INFO:teuthology.orchestra.run.smithi066.stdout: ceph x86_64 2:16.2.4-0.el8 ceph 6.3 k 2023-11-17T05:16:10.863 INFO:teuthology.orchestra.run.smithi066.stdout:Installing dependencies: 2023-11-17T05:16:10.863 INFO:teuthology.orchestra.run.smithi066.stdout: ceph-mds x86_64 2:16.2.4-0.el8 ceph 2.4 M 2023-11-17T05:16:10.863 INFO:teuthology.orchestra.run.smithi066.stdout: ceph-mgr x86_64 2:16.2.4-0.el8 ceph 1.6 M 2023-11-17T05:16:10.863 INFO:teuthology.orchestra.run.smithi066.stdout: ceph-mgr-modules-core noarch 2:16.2.4-0.el8 ceph-noarch 220 k 2023-11-17T05:16:10.863 INFO:teuthology.orchestra.run.smithi066.stdout: ceph-mon x86_64 2:16.2.4-0.el8 ceph 5.0 M 2023-11-17T05:16:10.863 INFO:teuthology.orchestra.run.smithi066.stdout: ceph-osd x86_64 2:16.2.4-0.el8 ceph 18 M 2023-11-17T05:16:10.863 INFO:teuthology.orchestra.run.smithi066.stdout: python3-bcrypt x86_64 3.1.6-2.el8.1 epel 44 k 2023-11-17T05:16:10.863 INFO:teuthology.orchestra.run.smithi066.stdout: python3-beautifulsoup4 noarch 4.6.3-2.el8.1 epel 185 k 2023-11-17T05:16:10.863 INFO:teuthology.orchestra.run.smithi066.stdout: python3-cffi x86_64 1.11.5-6.el8 baseos 249 k 2023-11-17T05:16:10.863 INFO:teuthology.orchestra.run.smithi066.stdout: python3-cheroot noarch 8.5.2-1.el8 epel 173 k 2023-11-17T05:16:10.864 INFO:teuthology.orchestra.run.smithi066.stdout: python3-cherrypy noarch 18.4.0-1.el8 epel 384 k 2023-11-17T05:16:10.864 INFO:teuthology.orchestra.run.smithi066.stdout: python3-cryptography x86_64 3.2.1-6.el8 baseos 641 k 2023-11-17T05:16:10.864 INFO:teuthology.orchestra.run.smithi066.stdout: python3-jaraco noarch 6.2-6.el8 epel 11 k 2023-11-17T05:16:10.864 INFO:teuthology.orchestra.run.smithi066.stdout: python3-jaraco-functools noarch 2.0-4.el8 epel 18 k 2023-11-17T05:16:10.864 INFO:teuthology.orchestra.run.smithi066.stdout: python3-logutils noarch 0.3.5-11.el8 epel 49 k 2023-11-17T05:16:10.864 INFO:teuthology.orchestra.run.smithi066.stdout: python3-mako noarch 1.0.6-14.el8 appstream 157 k 2023-11-17T05:16:10.864 INFO:teuthology.orchestra.run.smithi066.stdout: python3-markupsafe x86_64 0.23-19.el8 appstream 39 k 2023-11-17T05:16:10.864 INFO:teuthology.orchestra.run.smithi066.stdout: python3-more-itertools noarch 7.2.0-3.el8 epel 59 k 2023-11-17T05:16:10.864 INFO:teuthology.orchestra.run.smithi066.stdout: python3-pecan noarch 1.3.2-9.el8 epel 283 k 2023-11-17T05:16:10.864 INFO:teuthology.orchestra.run.smithi066.stdout: python3-portend noarch 2.6-1.el8 epel 16 k 2023-11-17T05:16:10.864 INFO:teuthology.orchestra.run.smithi066.stdout: python3-pyOpenSSL noarch 19.0.0-1.el8 appstream 103 k 2023-11-17T05:16:10.864 INFO:teuthology.orchestra.run.smithi066.stdout: python3-pycparser noarch 2.14-14.el8 baseos 109 k 2023-11-17T05:16:10.864 INFO:teuthology.orchestra.run.smithi066.stdout: python3-pytz noarch 2017.2-11.el8 appstream 57 k 2023-11-17T05:16:10.865 INFO:teuthology.orchestra.run.smithi066.stdout: python3-simplegeneric noarch 0.8.1-17.el8 epel 19 k 2023-11-17T05:16:10.865 INFO:teuthology.orchestra.run.smithi066.stdout: python3-singledispatch noarch 3.4.0.3-18.el8 epel 24 k 2023-11-17T05:16:10.865 INFO:teuthology.orchestra.run.smithi066.stdout: python3-tempora noarch 1.14.1-5.el8 epel 29 k 2023-11-17T05:16:10.865 INFO:teuthology.orchestra.run.smithi066.stdout: python3-trustme noarch 0.6.0-4.el8 epel 27 k 2023-11-17T05:16:10.865 INFO:teuthology.orchestra.run.smithi066.stdout: python3-waitress noarch 1.4.3-1.el8 epel 255 k 2023-11-17T05:16:10.865 INFO:teuthology.orchestra.run.smithi066.stdout: python3-webob noarch 1.8.5-1.el8.1 epel 251 k 2023-11-17T05:16:10.865 INFO:teuthology.orchestra.run.smithi066.stdout: python3-webtest noarch 2.0.33-1.el8 epel 85 k 2023-11-17T05:16:10.865 INFO:teuthology.orchestra.run.smithi066.stdout: python3-werkzeug noarch 0.12.2-4.el8 appstream 457 k 2023-11-17T05:16:10.865 INFO:teuthology.orchestra.run.smithi066.stdout: python3-zc-lockfile noarch 2.0-2.el8 epel 23 k 2023-11-17T05:16:10.865 INFO:teuthology.orchestra.run.smithi066.stdout: 2023-11-17T05:16:10.865 INFO:teuthology.orchestra.run.smithi066.stdout:Transaction Summary 2023-11-17T05:16:10.865 INFO:teuthology.orchestra.run.smithi066.stdout:================================================================================ 2023-11-17T05:16:10.865 INFO:teuthology.orchestra.run.smithi066.stdout:Install 32 Packages 2023-11-17T05:16:10.866 INFO:teuthology.orchestra.run.smithi066.stdout: 2023-11-17T05:16:10.866 INFO:teuthology.orchestra.run.smithi066.stdout:Total download size: 31 M 2023-11-17T05:16:10.866 INFO:teuthology.orchestra.run.smithi066.stdout:Installed size: 111 M 2023-11-17T05:16:10.866 INFO:teuthology.orchestra.run.smithi066.stdout:Downloading Packages: 2023-11-17T05:16:11.144 INFO:teuthology.orchestra.run.smithi066.stdout:(1/32): ceph-16.2.4-0.el8.x86_64.rpm 109 kB/s | 6.3 kB 00:00 2023-11-17T05:16:11.195 INFO:teuthology.orchestra.run.smithi066.stdout:(2/32): ceph-mgr-16.2.4-0.el8.x86_64.rpm 14 MB/s | 1.6 MB 00:00 2023-11-17T05:16:11.253 INFO:teuthology.orchestra.run.smithi066.stdout:(3/32): ceph-mds-16.2.4-0.el8.x86_64.rpm 14 MB/s | 2.4 MB 00:00 2023-11-17T05:16:11.289 INFO:teuthology.orchestra.run.smithi066.stdout:(4/32): ceph-mgr-modules-core-16.2.4-0.el8.noar 6.5 MB/s | 220 kB 00:00 2023-11-17T05:16:11.363 INFO:teuthology.orchestra.run.smithi066.stdout:(5/32): ceph-mon-16.2.4-0.el8.x86_64.rpm 23 MB/s | 5.0 MB 00:00 2023-11-17T05:16:11.596 INFO:teuthology.orchestra.run.smithi066.stdout:(6/32): ceph-osd-16.2.4-0.el8.x86_64.rpm 45 MB/s | 18 MB 00:00 2023-11-17T05:16:11.621 INFO:teuthology.orchestra.run.smithi066.stdout:(7/32): python3-markupsafe-0.23-19.el8.x86_64.r 150 kB/s | 39 kB 00:00 2023-11-17T05:16:11.647 INFO:teuthology.orchestra.run.smithi066.stdout:(8/32): python3-mako-1.0.6-14.el8.noarch.rpm 437 kB/s | 157 kB 00:00 2023-11-17T05:16:11.705 INFO:teuthology.orchestra.run.smithi066.stdout:(9/32): python3-pytz-2017.2-11.el8.noarch.rpm 681 kB/s | 57 kB 00:00 2023-11-17T05:16:11.826 INFO:teuthology.orchestra.run.smithi066.stdout:(10/32): python3-werkzeug-0.12.2-4.el8.noarch.r 2.5 MB/s | 457 kB 00:00 2023-11-17T05:16:11.856 INFO:teuthology.orchestra.run.smithi066.stdout:(11/32): python3-pyOpenSSL-19.0.0-1.el8.noarch. 399 kB/s | 103 kB 00:00 2023-11-17T05:16:11.874 INFO:teuthology.orchestra.run.smithi040.stdout:Dependencies resolved. 2023-11-17T05:16:11.875 INFO:teuthology.orchestra.run.smithi040.stdout:================================================================================ 2023-11-17T05:16:11.875 INFO:teuthology.orchestra.run.smithi040.stdout: Package Architecture Version Repository Size 2023-11-17T05:16:11.875 INFO:teuthology.orchestra.run.smithi040.stdout:================================================================================ 2023-11-17T05:16:11.876 INFO:teuthology.orchestra.run.smithi040.stdout:Installing: 2023-11-17T05:16:11.876 INFO:teuthology.orchestra.run.smithi040.stdout: ceph-test x86_64 2:16.2.4-0.el8 ceph 52 M 2023-11-17T05:16:11.876 INFO:teuthology.orchestra.run.smithi040.stdout:Installing dependencies: 2023-11-17T05:16:11.876 INFO:teuthology.orchestra.run.smithi040.stdout: jq x86_64 1.6-8.el8 appstream 203 k 2023-11-17T05:16:11.876 INFO:teuthology.orchestra.run.smithi040.stdout: libcephsqlite x86_64 2:16.2.4-0.el8 ceph 169 k 2023-11-17T05:16:11.876 INFO:teuthology.orchestra.run.smithi040.stdout: oniguruma x86_64 6.8.2-2.el8 appstream 187 k 2023-11-17T05:16:11.876 INFO:teuthology.orchestra.run.smithi040.stdout: socat x86_64 1.7.4.1-1.el8 appstream 323 k 2023-11-17T05:16:11.876 INFO:teuthology.orchestra.run.smithi040.stdout: xmlstarlet x86_64 1.6.1-20.el8 appstream 71 k 2023-11-17T05:16:11.876 INFO:teuthology.orchestra.run.smithi040.stdout: 2023-11-17T05:16:11.876 INFO:teuthology.orchestra.run.smithi040.stdout:Transaction Summary 2023-11-17T05:16:11.876 INFO:teuthology.orchestra.run.smithi040.stdout:================================================================================ 2023-11-17T05:16:11.876 INFO:teuthology.orchestra.run.smithi040.stdout:Install 6 Packages 2023-11-17T05:16:11.876 INFO:teuthology.orchestra.run.smithi040.stdout: 2023-11-17T05:16:11.877 INFO:teuthology.orchestra.run.smithi040.stdout:Total download size: 53 M 2023-11-17T05:16:11.877 INFO:teuthology.orchestra.run.smithi040.stdout:Installed size: 229 M 2023-11-17T05:16:11.877 INFO:teuthology.orchestra.run.smithi040.stdout:Downloading Packages: 2023-11-17T05:16:11.881 INFO:teuthology.orchestra.run.smithi066.stdout:(12/32): python3-cffi-1.11.5-6.el8.x86_64.rpm 1.4 MB/s | 249 kB 00:00 2023-11-17T05:16:11.973 INFO:teuthology.orchestra.run.smithi066.stdout:(13/32): python3-pycparser-2.14-14.el8.noarch.r 934 kB/s | 109 kB 00:00 2023-11-17T05:16:12.006 INFO:teuthology.orchestra.run.smithi066.stdout:(14/32): python3-cryptography-3.2.1-6.el8.x86_6 3.5 MB/s | 641 kB 00:00 2023-11-17T05:16:12.038 INFO:teuthology.orchestra.run.smithi066.stdout:(15/32): python3-bcrypt-3.1.6-2.el8.1.x86_64.rp 290 kB/s | 44 kB 00:00 2023-11-17T05:16:12.133 INFO:teuthology.orchestra.run.smithi040.stdout:(1/6): libcephsqlite-16.2.4-0.el8.x86_64.rpm 2.6 MB/s | 169 kB 00:00 2023-11-17T05:16:12.157 INFO:teuthology.orchestra.run.smithi066.stdout:(16/32): python3-cherrypy-18.4.0-1.el8.noarch.r 3.1 MB/s | 384 kB 00:00 2023-11-17T05:16:12.199 INFO:teuthology.orchestra.run.smithi066.stdout:(17/32): python3-cheroot-8.5.2-1.el8.noarch.rpm 922 kB/s | 173 kB 00:00 2023-11-17T05:16:12.224 INFO:teuthology.orchestra.run.smithi066.stdout:(18/32): python3-beautifulsoup4-4.6.3-2.el8.1.n 738 kB/s | 185 kB 00:00 2023-11-17T05:16:12.241 INFO:teuthology.orchestra.run.smithi066.stdout:(19/32): python3-jaraco-6.2-6.el8.noarch.rpm 133 kB/s | 11 kB 00:00 2023-11-17T05:16:12.275 INFO:teuthology.orchestra.run.smithi066.stdout:(20/32): python3-jaraco-functools-2.0-4.el8.noa 233 kB/s | 18 kB 00:00 2023-11-17T05:16:12.321 INFO:teuthology.orchestra.run.smithi066.stdout:(21/32): python3-logutils-0.3.5-11.el8.noarch.r 613 kB/s | 49 kB 00:00 2023-11-17T05:16:12.325 INFO:teuthology.orchestra.run.smithi040.stdout:(2/6): jq-1.6-8.el8.x86_64.rpm 795 kB/s | 203 kB 00:00 2023-11-17T05:16:12.350 INFO:teuthology.orchestra.run.smithi040.stdout:(3/6): oniguruma-6.8.2-2.el8.x86_64.rpm 865 kB/s | 187 kB 00:00 2023-11-17T05:16:12.359 INFO:teuthology.orchestra.run.smithi066.stdout:(22/32): python3-more-itertools-7.2.0-3.el8.noa 500 kB/s | 59 kB 00:00 2023-11-17T05:16:12.384 INFO:teuthology.orchestra.run.smithi066.stdout:(23/32): python3-pecan-1.3.2-9.el8.noarch.rpm 2.5 MB/s | 283 kB 00:00 2023-11-17T05:16:12.401 INFO:teuthology.orchestra.run.smithi066.stdout:(24/32): python3-portend-2.6-1.el8.noarch.rpm 372 kB/s | 16 kB 00:00 2023-11-17T05:16:12.409 INFO:teuthology.orchestra.run.smithi040.stdout:(4/6): xmlstarlet-1.6.1-20.el8.x86_64.rpm 1.2 MB/s | 71 kB 00:00 2023-11-17T05:16:12.418 INFO:teuthology.orchestra.run.smithi066.stdout:(25/32): python3-simplegeneric-0.8.1-17.el8.noa 319 kB/s | 19 kB 00:00 2023-11-17T05:16:12.435 INFO:teuthology.orchestra.run.smithi066.stdout:(26/32): python3-singledispatch-3.4.0.3-18.el8. 472 kB/s | 24 kB 00:00 2023-11-17T05:16:12.452 INFO:teuthology.orchestra.run.smithi066.stdout:(27/32): python3-tempora-1.14.1-5.el8.noarch.rp 581 kB/s | 29 kB 00:00 2023-11-17T05:16:12.469 INFO:teuthology.orchestra.run.smithi066.stdout:(28/32): python3-trustme-0.6.0-4.el8.noarch.rpm 544 kB/s | 27 kB 00:00 2023-11-17T05:16:12.476 INFO:teuthology.orchestra.run.smithi040.stdout:(5/6): socat-1.7.4.1-1.el8.x86_64.rpm 2.1 MB/s | 323 kB 00:00 2023-11-17T05:16:12.502 INFO:teuthology.orchestra.run.smithi066.stdout:(29/32): python3-waitress-1.4.3-1.el8.noarch.rp 3.7 MB/s | 255 kB 00:00 2023-11-17T05:16:12.553 INFO:teuthology.orchestra.run.smithi066.stdout:(30/32): python3-webob-1.8.5-1.el8.1.noarch.rpm 2.4 MB/s | 251 kB 00:00 2023-11-17T05:16:12.611 INFO:teuthology.orchestra.run.smithi066.stdout:(31/32): python3-webtest-2.0.33-1.el8.noarch.rp 601 kB/s | 85 kB 00:00 2023-11-17T05:16:12.636 INFO:teuthology.orchestra.run.smithi066.stdout:(32/32): python3-zc-lockfile-2.0-2.el8.noarch.r 171 kB/s | 23 kB 00:00 2023-11-17T05:16:12.640 INFO:teuthology.orchestra.run.smithi066.stdout:-------------------------------------------------------------------------------- 2023-11-17T05:16:12.640 INFO:teuthology.orchestra.run.smithi066.stdout:Total 17 MB/s | 31 MB 00:01 2023-11-17T05:16:12.722 INFO:teuthology.orchestra.run.smithi066.stdout:Running transaction check 2023-11-17T05:16:12.763 INFO:teuthology.orchestra.run.smithi066.stdout:Transaction check succeeded. 2023-11-17T05:16:12.763 INFO:teuthology.orchestra.run.smithi066.stdout:Running transaction test 2023-11-17T05:16:12.976 INFO:teuthology.orchestra.run.smithi040.stdout:(6/6): ceph-test-16.2.4-0.el8.x86_64.rpm 57 MB/s | 52 MB 00:00 2023-11-17T05:16:12.978 INFO:teuthology.orchestra.run.smithi040.stdout:-------------------------------------------------------------------------------- 2023-11-17T05:16:12.978 INFO:teuthology.orchestra.run.smithi040.stdout:Total 48 MB/s | 53 MB 00:01 2023-11-17T05:16:13.026 INFO:teuthology.orchestra.run.smithi040.stdout:Running transaction check 2023-11-17T05:16:13.049 INFO:teuthology.orchestra.run.smithi040.stdout:Transaction check succeeded. 2023-11-17T05:16:13.049 INFO:teuthology.orchestra.run.smithi040.stdout:Running transaction test 2023-11-17T05:16:13.092 INFO:teuthology.orchestra.run.smithi066.stdout:Transaction test succeeded. 2023-11-17T05:16:13.095 INFO:teuthology.orchestra.run.smithi066.stdout:Running transaction 2023-11-17T05:16:13.390 INFO:teuthology.orchestra.run.smithi040.stdout:Transaction test succeeded. 2023-11-17T05:16:13.394 INFO:teuthology.orchestra.run.smithi040.stdout:Running transaction 2023-11-17T05:16:13.900 INFO:teuthology.orchestra.run.smithi040.stdout: Preparing : 1/1 2023-11-17T05:16:14.128 INFO:teuthology.orchestra.run.smithi040.stdout: Installing : xmlstarlet-1.6.1-20.el8.x86_64 1/6 2023-11-17T05:16:14.247 INFO:teuthology.orchestra.run.smithi066.stdout: Preparing : 1/1 2023-11-17T05:16:14.316 INFO:teuthology.orchestra.run.smithi040.stdout: Installing : socat-1.7.4.1-1.el8.x86_64 2/6 2023-11-17T05:16:14.484 INFO:teuthology.orchestra.run.smithi066.stdout: Installing : python3-more-itertools-7.2.0-3.el8.noarch 1/32 2023-11-17T05:16:14.527 INFO:teuthology.orchestra.run.smithi040.stdout: Installing : oniguruma-6.8.2-2.el8.x86_64 3/6 2023-11-17T05:16:14.600 INFO:teuthology.orchestra.run.smithi040.stdout: Running scriptlet: oniguruma-6.8.2-2.el8.x86_64 3/6 2023-11-17T05:16:14.683 INFO:teuthology.orchestra.run.smithi066.stdout: Installing : python3-webob-1.8.5-1.el8.1.noarch 2/32 2023-11-17T05:16:14.774 INFO:teuthology.orchestra.run.smithi040.stdout: Installing : jq-1.6-8.el8.x86_64 4/6 2023-11-17T05:16:14.919 INFO:teuthology.orchestra.run.smithi040.stdout: Installing : libcephsqlite-2:16.2.4-0.el8.x86_64 5/6 2023-11-17T05:16:14.960 INFO:teuthology.orchestra.run.smithi066.stdout: Installing : python3-zc-lockfile-2.0-2.el8.noarch 3/32 2023-11-17T05:16:15.147 INFO:teuthology.orchestra.run.smithi066.stdout: Installing : python3-waitress-1.4.3-1.el8.noarch 4/32 2023-11-17T05:16:15.345 INFO:teuthology.orchestra.run.smithi066.stdout: Installing : python3-singledispatch-3.4.0.3-18.el8.noarch 5/32 2023-11-17T05:16:15.500 INFO:teuthology.orchestra.run.smithi066.stdout: Installing : python3-simplegeneric-0.8.1-17.el8.noarch 6/32 2023-11-17T05:16:15.660 INFO:teuthology.orchestra.run.smithi066.stdout: Installing : python3-logutils-0.3.5-11.el8.noarch 7/32 2023-11-17T05:16:15.854 INFO:teuthology.orchestra.run.smithi066.stdout: Installing : python3-jaraco-6.2-6.el8.noarch 8/32 2023-11-17T05:16:16.056 INFO:teuthology.orchestra.run.smithi066.stdout: Installing : python3-jaraco-functools-2.0-4.el8.noarch 9/32 2023-11-17T05:16:16.328 INFO:teuthology.orchestra.run.smithi066.stdout: Installing : python3-beautifulsoup4-4.6.3-2.el8.1.noarch 10/32 2023-11-17T05:16:16.561 INFO:teuthology.orchestra.run.smithi066.stdout: Installing : python3-webtest-2.0.33-1.el8.noarch 11/32 2023-11-17T05:16:16.842 INFO:teuthology.orchestra.run.smithi066.stdout: Installing : python3-pycparser-2.14-14.el8.noarch 12/32 2023-11-17T05:16:17.226 INFO:teuthology.orchestra.run.smithi066.stdout: Installing : python3-cffi-1.11.5-6.el8.x86_64 13/32 2023-11-17T05:16:17.511 INFO:teuthology.orchestra.run.smithi066.stdout: Installing : python3-cryptography-3.2.1-6.el8.x86_64 14/32 2023-11-17T05:16:17.665 INFO:teuthology.orchestra.run.smithi066.stdout: Installing : python3-pyOpenSSL-19.0.0-1.el8.noarch 15/32 2023-11-17T05:16:17.919 INFO:teuthology.orchestra.run.smithi066.stdout: Installing : python3-trustme-0.6.0-4.el8.noarch 16/32 2023-11-17T05:16:18.116 INFO:teuthology.orchestra.run.smithi066.stdout: Installing : python3-cheroot-8.5.2-1.el8.noarch 17/32 2023-11-17T05:16:18.481 INFO:teuthology.orchestra.run.smithi066.stdout: Installing : python3-bcrypt-3.1.6-2.el8.1.x86_64 18/32 2023-11-17T05:16:18.694 INFO:teuthology.orchestra.run.smithi066.stdout: Installing : python3-werkzeug-0.12.2-4.el8.noarch 19/32 2023-11-17T05:16:18.885 INFO:teuthology.orchestra.run.smithi066.stdout: Installing : python3-pytz-2017.2-11.el8.noarch 20/32 2023-11-17T05:16:19.076 INFO:teuthology.orchestra.run.smithi066.stdout: Installing : python3-tempora-1.14.1-5.el8.noarch 21/32 2023-11-17T05:16:19.327 INFO:teuthology.orchestra.run.smithi066.stdout: Installing : python3-portend-2.6-1.el8.noarch 22/32 2023-11-17T05:16:19.561 INFO:teuthology.orchestra.run.smithi066.stdout: Installing : python3-cherrypy-18.4.0-1.el8.noarch 23/32 2023-11-17T05:16:19.764 INFO:teuthology.orchestra.run.smithi066.stdout: Installing : python3-markupsafe-0.23-19.el8.x86_64 24/32 2023-11-17T05:16:20.103 INFO:teuthology.orchestra.run.smithi066.stdout: Installing : python3-mako-1.0.6-14.el8.noarch 25/32 2023-11-17T05:16:20.287 INFO:teuthology.orchestra.run.smithi040.stdout: Running scriptlet: libcephsqlite-2:16.2.4-0.el8.x86_64 5/6 2023-11-17T05:16:20.417 INFO:teuthology.orchestra.run.smithi066.stdout: Installing : python3-pecan-1.3.2-9.el8.noarch 26/32 2023-11-17T05:16:20.535 INFO:teuthology.orchestra.run.smithi040.stdout: Installing : ceph-test-2:16.2.4-0.el8.x86_64 6/6 2023-11-17T05:16:20.832 INFO:teuthology.orchestra.run.smithi066.stdout: Installing : ceph-mgr-modules-core-2:16.2.4-0.el8.noarch 27/32 2023-11-17T05:16:20.861 INFO:teuthology.orchestra.run.smithi040.stdout: Running scriptlet: ceph-test-2:16.2.4-0.el8.x86_64 6/6 2023-11-17T05:16:20.861 INFO:teuthology.orchestra.run.smithi040.stdout: Verifying : ceph-test-2:16.2.4-0.el8.x86_64 1/6 2023-11-17T05:16:20.862 INFO:teuthology.orchestra.run.smithi040.stdout: Verifying : libcephsqlite-2:16.2.4-0.el8.x86_64 2/6 2023-11-17T05:16:20.862 INFO:teuthology.orchestra.run.smithi040.stdout: Verifying : jq-1.6-8.el8.x86_64 3/6 2023-11-17T05:16:20.862 INFO:teuthology.orchestra.run.smithi040.stdout: Verifying : oniguruma-6.8.2-2.el8.x86_64 4/6 2023-11-17T05:16:20.862 INFO:teuthology.orchestra.run.smithi040.stdout: Verifying : socat-1.7.4.1-1.el8.x86_64 5/6 2023-11-17T05:16:21.019 INFO:teuthology.orchestra.run.smithi066.stdout: Installing : ceph-mgr-2:16.2.4-0.el8.x86_64 28/32 2023-11-17T05:16:21.663 INFO:teuthology.orchestra.run.smithi040.stdout: Verifying : xmlstarlet-1.6.1-20.el8.x86_64 6/6 2023-11-17T05:16:21.663 INFO:teuthology.orchestra.run.smithi040.stdout: 2023-11-17T05:16:21.663 INFO:teuthology.orchestra.run.smithi040.stdout:Installed: 2023-11-17T05:16:21.664 INFO:teuthology.orchestra.run.smithi040.stdout: ceph-test-2:16.2.4-0.el8.x86_64 jq-1.6-8.el8.x86_64 2023-11-17T05:16:21.664 INFO:teuthology.orchestra.run.smithi040.stdout: libcephsqlite-2:16.2.4-0.el8.x86_64 oniguruma-6.8.2-2.el8.x86_64 2023-11-17T05:16:21.664 INFO:teuthology.orchestra.run.smithi040.stdout: socat-1.7.4.1-1.el8.x86_64 xmlstarlet-1.6.1-20.el8.x86_64 2023-11-17T05:16:21.664 INFO:teuthology.orchestra.run.smithi040.stdout: 2023-11-17T05:16:21.664 INFO:teuthology.orchestra.run.smithi040.stdout:Complete! 2023-11-17T05:16:21.940 DEBUG:teuthology.orchestra.run.smithi040:> sudo yum -y install ceph 2023-11-17T05:16:22.649 INFO:teuthology.orchestra.run.smithi066.stdout: Running scriptlet: ceph-mgr-2:16.2.4-0.el8.x86_64 28/32 2023-11-17T05:16:22.844 INFO:teuthology.orchestra.run.smithi066.stdout: Installing : ceph-osd-2:16.2.4-0.el8.x86_64 29/32 2023-11-17T05:16:22.921 INFO:teuthology.orchestra.run.smithi040.stdout:Last metadata expiration check: 0:01:04 ago on Fri 17 Nov 2023 05:15:18 AM UTC. 2023-11-17T05:16:23.304 INFO:teuthology.orchestra.run.smithi066.stdout: Running scriptlet: ceph-osd-2:16.2.4-0.el8.x86_64 29/32 2023-11-17T05:16:23.502 INFO:teuthology.orchestra.run.smithi066.stdout: Installing : ceph-mon-2:16.2.4-0.el8.x86_64 30/32 2023-11-17T05:16:23.737 INFO:teuthology.orchestra.run.smithi066.stdout: Running scriptlet: ceph-mon-2:16.2.4-0.el8.x86_64 30/32 2023-11-17T05:16:23.915 INFO:teuthology.orchestra.run.smithi066.stdout: Installing : ceph-mds-2:16.2.4-0.el8.x86_64 31/32 2023-11-17T05:16:23.933 INFO:teuthology.orchestra.run.smithi066.stdout: Running scriptlet: ceph-mds-2:16.2.4-0.el8.x86_64 31/32 2023-11-17T05:16:24.073 INFO:teuthology.orchestra.run.smithi066.stdout: Installing : ceph-2:16.2.4-0.el8.x86_64 32/32 2023-11-17T05:16:24.498 INFO:teuthology.orchestra.run.smithi066.stdout: Running scriptlet: ceph-2:16.2.4-0.el8.x86_64 32/32 2023-11-17T05:16:24.499 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : ceph-2:16.2.4-0.el8.x86_64 1/32 2023-11-17T05:16:24.499 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : ceph-mds-2:16.2.4-0.el8.x86_64 2/32 2023-11-17T05:16:24.499 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : ceph-mgr-2:16.2.4-0.el8.x86_64 3/32 2023-11-17T05:16:24.499 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : ceph-mon-2:16.2.4-0.el8.x86_64 4/32 2023-11-17T05:16:24.499 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : ceph-osd-2:16.2.4-0.el8.x86_64 5/32 2023-11-17T05:16:24.499 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : ceph-mgr-modules-core-2:16.2.4-0.el8.noarch 6/32 2023-11-17T05:16:24.499 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : python3-mako-1.0.6-14.el8.noarch 7/32 2023-11-17T05:16:24.499 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : python3-markupsafe-0.23-19.el8.x86_64 8/32 2023-11-17T05:16:24.499 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : python3-pyOpenSSL-19.0.0-1.el8.noarch 9/32 2023-11-17T05:16:24.499 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : python3-pytz-2017.2-11.el8.noarch 10/32 2023-11-17T05:16:24.499 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : python3-werkzeug-0.12.2-4.el8.noarch 11/32 2023-11-17T05:16:24.500 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : python3-cffi-1.11.5-6.el8.x86_64 12/32 2023-11-17T05:16:24.500 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : python3-cryptography-3.2.1-6.el8.x86_64 13/32 2023-11-17T05:16:24.500 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : python3-pycparser-2.14-14.el8.noarch 14/32 2023-11-17T05:16:24.500 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : python3-bcrypt-3.1.6-2.el8.1.x86_64 15/32 2023-11-17T05:16:24.500 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : python3-beautifulsoup4-4.6.3-2.el8.1.noarch 16/32 2023-11-17T05:16:24.500 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : python3-cheroot-8.5.2-1.el8.noarch 17/32 2023-11-17T05:16:24.500 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : python3-cherrypy-18.4.0-1.el8.noarch 18/32 2023-11-17T05:16:24.500 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : python3-jaraco-6.2-6.el8.noarch 19/32 2023-11-17T05:16:24.500 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : python3-jaraco-functools-2.0-4.el8.noarch 20/32 2023-11-17T05:16:24.500 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : python3-logutils-0.3.5-11.el8.noarch 21/32 2023-11-17T05:16:24.500 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : python3-more-itertools-7.2.0-3.el8.noarch 22/32 2023-11-17T05:16:24.500 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : python3-pecan-1.3.2-9.el8.noarch 23/32 2023-11-17T05:16:24.500 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : python3-portend-2.6-1.el8.noarch 24/32 2023-11-17T05:16:24.501 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : python3-simplegeneric-0.8.1-17.el8.noarch 25/32 2023-11-17T05:16:24.501 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : python3-singledispatch-3.4.0.3-18.el8.noarch 26/32 2023-11-17T05:16:24.501 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : python3-tempora-1.14.1-5.el8.noarch 27/32 2023-11-17T05:16:24.501 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : python3-trustme-0.6.0-4.el8.noarch 28/32 2023-11-17T05:16:24.501 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : python3-waitress-1.4.3-1.el8.noarch 29/32 2023-11-17T05:16:24.501 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : python3-webob-1.8.5-1.el8.1.noarch 30/32 2023-11-17T05:16:24.501 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : python3-webtest-2.0.33-1.el8.noarch 31/32 2023-11-17T05:16:24.786 INFO:teuthology.orchestra.run.smithi040.stdout:Dependencies resolved. 2023-11-17T05:16:24.790 INFO:teuthology.orchestra.run.smithi040.stdout:================================================================================ 2023-11-17T05:16:24.790 INFO:teuthology.orchestra.run.smithi040.stdout: Package Arch Version Repository Size 2023-11-17T05:16:24.790 INFO:teuthology.orchestra.run.smithi040.stdout:================================================================================ 2023-11-17T05:16:24.790 INFO:teuthology.orchestra.run.smithi040.stdout:Installing: 2023-11-17T05:16:24.790 INFO:teuthology.orchestra.run.smithi040.stdout: ceph x86_64 2:16.2.4-0.el8 ceph 6.3 k 2023-11-17T05:16:24.790 INFO:teuthology.orchestra.run.smithi040.stdout:Installing dependencies: 2023-11-17T05:16:24.790 INFO:teuthology.orchestra.run.smithi040.stdout: ceph-mds x86_64 2:16.2.4-0.el8 ceph 2.4 M 2023-11-17T05:16:24.790 INFO:teuthology.orchestra.run.smithi040.stdout: ceph-mgr x86_64 2:16.2.4-0.el8 ceph 1.6 M 2023-11-17T05:16:24.790 INFO:teuthology.orchestra.run.smithi040.stdout: ceph-mgr-modules-core noarch 2:16.2.4-0.el8 ceph-noarch 220 k 2023-11-17T05:16:24.791 INFO:teuthology.orchestra.run.smithi040.stdout: ceph-mon x86_64 2:16.2.4-0.el8 ceph 5.0 M 2023-11-17T05:16:24.791 INFO:teuthology.orchestra.run.smithi040.stdout: ceph-osd x86_64 2:16.2.4-0.el8 ceph 18 M 2023-11-17T05:16:24.791 INFO:teuthology.orchestra.run.smithi040.stdout: python3-bcrypt x86_64 3.1.6-2.el8.1 epel 44 k 2023-11-17T05:16:24.791 INFO:teuthology.orchestra.run.smithi040.stdout: python3-beautifulsoup4 noarch 4.6.3-2.el8.1 epel 185 k 2023-11-17T05:16:24.791 INFO:teuthology.orchestra.run.smithi040.stdout: python3-cffi x86_64 1.11.5-6.el8 baseos 249 k 2023-11-17T05:16:24.791 INFO:teuthology.orchestra.run.smithi040.stdout: python3-cheroot noarch 8.5.2-1.el8 epel 173 k 2023-11-17T05:16:24.791 INFO:teuthology.orchestra.run.smithi040.stdout: python3-cherrypy noarch 18.4.0-1.el8 epel 384 k 2023-11-17T05:16:24.791 INFO:teuthology.orchestra.run.smithi040.stdout: python3-cryptography x86_64 3.2.1-6.el8 baseos 641 k 2023-11-17T05:16:24.791 INFO:teuthology.orchestra.run.smithi040.stdout: python3-jaraco noarch 6.2-6.el8 epel 11 k 2023-11-17T05:16:24.791 INFO:teuthology.orchestra.run.smithi040.stdout: python3-jaraco-functools noarch 2.0-4.el8 epel 18 k 2023-11-17T05:16:24.791 INFO:teuthology.orchestra.run.smithi040.stdout: python3-logutils noarch 0.3.5-11.el8 epel 49 k 2023-11-17T05:16:24.791 INFO:teuthology.orchestra.run.smithi040.stdout: python3-mako noarch 1.0.6-14.el8 appstream 157 k 2023-11-17T05:16:24.792 INFO:teuthology.orchestra.run.smithi040.stdout: python3-markupsafe x86_64 0.23-19.el8 appstream 39 k 2023-11-17T05:16:24.792 INFO:teuthology.orchestra.run.smithi040.stdout: python3-more-itertools noarch 7.2.0-3.el8 epel 59 k 2023-11-17T05:16:24.792 INFO:teuthology.orchestra.run.smithi040.stdout: python3-pecan noarch 1.3.2-9.el8 epel 283 k 2023-11-17T05:16:24.792 INFO:teuthology.orchestra.run.smithi040.stdout: python3-portend noarch 2.6-1.el8 epel 16 k 2023-11-17T05:16:24.792 INFO:teuthology.orchestra.run.smithi040.stdout: python3-pyOpenSSL noarch 19.0.0-1.el8 appstream 103 k 2023-11-17T05:16:24.792 INFO:teuthology.orchestra.run.smithi040.stdout: python3-pycparser noarch 2.14-14.el8 baseos 109 k 2023-11-17T05:16:24.792 INFO:teuthology.orchestra.run.smithi040.stdout: python3-pytz noarch 2017.2-11.el8 appstream 57 k 2023-11-17T05:16:24.792 INFO:teuthology.orchestra.run.smithi040.stdout: python3-simplegeneric noarch 0.8.1-17.el8 epel 19 k 2023-11-17T05:16:24.792 INFO:teuthology.orchestra.run.smithi040.stdout: python3-singledispatch noarch 3.4.0.3-18.el8 epel 24 k 2023-11-17T05:16:24.792 INFO:teuthology.orchestra.run.smithi040.stdout: python3-tempora noarch 1.14.1-5.el8 epel 29 k 2023-11-17T05:16:24.792 INFO:teuthology.orchestra.run.smithi040.stdout: python3-trustme noarch 0.6.0-4.el8 epel 27 k 2023-11-17T05:16:24.792 INFO:teuthology.orchestra.run.smithi040.stdout: python3-waitress noarch 1.4.3-1.el8 epel 255 k 2023-11-17T05:16:24.792 INFO:teuthology.orchestra.run.smithi040.stdout: python3-webob noarch 1.8.5-1.el8.1 epel 251 k 2023-11-17T05:16:24.793 INFO:teuthology.orchestra.run.smithi040.stdout: python3-webtest noarch 2.0.33-1.el8 epel 85 k 2023-11-17T05:16:24.793 INFO:teuthology.orchestra.run.smithi040.stdout: python3-werkzeug noarch 0.12.2-4.el8 appstream 457 k 2023-11-17T05:16:24.793 INFO:teuthology.orchestra.run.smithi040.stdout: python3-zc-lockfile noarch 2.0-2.el8 epel 23 k 2023-11-17T05:16:24.793 INFO:teuthology.orchestra.run.smithi040.stdout: 2023-11-17T05:16:24.793 INFO:teuthology.orchestra.run.smithi040.stdout:Transaction Summary 2023-11-17T05:16:24.793 INFO:teuthology.orchestra.run.smithi040.stdout:================================================================================ 2023-11-17T05:16:24.793 INFO:teuthology.orchestra.run.smithi040.stdout:Install 32 Packages 2023-11-17T05:16:24.793 INFO:teuthology.orchestra.run.smithi040.stdout: 2023-11-17T05:16:24.794 INFO:teuthology.orchestra.run.smithi040.stdout:Total download size: 31 M 2023-11-17T05:16:24.794 INFO:teuthology.orchestra.run.smithi040.stdout:Installed size: 111 M 2023-11-17T05:16:24.794 INFO:teuthology.orchestra.run.smithi040.stdout:Downloading Packages: 2023-11-17T05:16:25.379 INFO:teuthology.orchestra.run.smithi040.stdout:(1/32): ceph-16.2.4-0.el8.x86_64.rpm 127 kB/s | 6.3 kB 00:00 2023-11-17T05:16:25.470 INFO:teuthology.orchestra.run.smithi040.stdout:(2/32): ceph-mgr-16.2.4-0.el8.x86_64.rpm 11 MB/s | 1.6 MB 00:00 2023-11-17T05:16:25.537 INFO:teuthology.orchestra.run.smithi040.stdout:(3/32): ceph-mds-16.2.4-0.el8.x86_64.rpm 11 MB/s | 2.4 MB 00:00 2023-11-17T05:16:25.554 INFO:teuthology.orchestra.run.smithi040.stdout:(4/32): ceph-mgr-modules-core-16.2.4-0.el8.noar 13 MB/s | 220 kB 00:00 2023-11-17T05:16:25.646 INFO:teuthology.orchestra.run.smithi040.stdout:(5/32): ceph-mon-16.2.4-0.el8.x86_64.rpm 19 MB/s | 5.0 MB 00:00 2023-11-17T05:16:25.681 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : python3-zc-lockfile-2.0-2.el8.noarch 32/32 2023-11-17T05:16:25.681 INFO:teuthology.orchestra.run.smithi066.stdout: 2023-11-17T05:16:25.681 INFO:teuthology.orchestra.run.smithi066.stdout:Installed: 2023-11-17T05:16:25.681 INFO:teuthology.orchestra.run.smithi066.stdout: ceph-2:16.2.4-0.el8.x86_64 2023-11-17T05:16:25.681 INFO:teuthology.orchestra.run.smithi066.stdout: ceph-mds-2:16.2.4-0.el8.x86_64 2023-11-17T05:16:25.681 INFO:teuthology.orchestra.run.smithi066.stdout: ceph-mgr-2:16.2.4-0.el8.x86_64 2023-11-17T05:16:25.681 INFO:teuthology.orchestra.run.smithi066.stdout: ceph-mgr-modules-core-2:16.2.4-0.el8.noarch 2023-11-17T05:16:25.681 INFO:teuthology.orchestra.run.smithi066.stdout: ceph-mon-2:16.2.4-0.el8.x86_64 2023-11-17T05:16:25.681 INFO:teuthology.orchestra.run.smithi066.stdout: ceph-osd-2:16.2.4-0.el8.x86_64 2023-11-17T05:16:25.681 INFO:teuthology.orchestra.run.smithi066.stdout: python3-bcrypt-3.1.6-2.el8.1.x86_64 2023-11-17T05:16:25.682 INFO:teuthology.orchestra.run.smithi066.stdout: python3-beautifulsoup4-4.6.3-2.el8.1.noarch 2023-11-17T05:16:25.682 INFO:teuthology.orchestra.run.smithi066.stdout: python3-cffi-1.11.5-6.el8.x86_64 2023-11-17T05:16:25.682 INFO:teuthology.orchestra.run.smithi066.stdout: python3-cheroot-8.5.2-1.el8.noarch 2023-11-17T05:16:25.682 INFO:teuthology.orchestra.run.smithi066.stdout: python3-cherrypy-18.4.0-1.el8.noarch 2023-11-17T05:16:25.682 INFO:teuthology.orchestra.run.smithi066.stdout: python3-cryptography-3.2.1-6.el8.x86_64 2023-11-17T05:16:25.682 INFO:teuthology.orchestra.run.smithi066.stdout: python3-jaraco-6.2-6.el8.noarch 2023-11-17T05:16:25.682 INFO:teuthology.orchestra.run.smithi066.stdout: python3-jaraco-functools-2.0-4.el8.noarch 2023-11-17T05:16:25.682 INFO:teuthology.orchestra.run.smithi066.stdout: python3-logutils-0.3.5-11.el8.noarch 2023-11-17T05:16:25.682 INFO:teuthology.orchestra.run.smithi066.stdout: python3-mako-1.0.6-14.el8.noarch 2023-11-17T05:16:25.682 INFO:teuthology.orchestra.run.smithi066.stdout: python3-markupsafe-0.23-19.el8.x86_64 2023-11-17T05:16:25.682 INFO:teuthology.orchestra.run.smithi066.stdout: python3-more-itertools-7.2.0-3.el8.noarch 2023-11-17T05:16:25.683 INFO:teuthology.orchestra.run.smithi066.stdout: python3-pecan-1.3.2-9.el8.noarch 2023-11-17T05:16:25.683 INFO:teuthology.orchestra.run.smithi066.stdout: python3-portend-2.6-1.el8.noarch 2023-11-17T05:16:25.683 INFO:teuthology.orchestra.run.smithi066.stdout: python3-pyOpenSSL-19.0.0-1.el8.noarch 2023-11-17T05:16:25.683 INFO:teuthology.orchestra.run.smithi066.stdout: python3-pycparser-2.14-14.el8.noarch 2023-11-17T05:16:25.683 INFO:teuthology.orchestra.run.smithi066.stdout: python3-pytz-2017.2-11.el8.noarch 2023-11-17T05:16:25.683 INFO:teuthology.orchestra.run.smithi066.stdout: python3-simplegeneric-0.8.1-17.el8.noarch 2023-11-17T05:16:25.683 INFO:teuthology.orchestra.run.smithi066.stdout: python3-singledispatch-3.4.0.3-18.el8.noarch 2023-11-17T05:16:25.683 INFO:teuthology.orchestra.run.smithi066.stdout: python3-tempora-1.14.1-5.el8.noarch 2023-11-17T05:16:25.683 INFO:teuthology.orchestra.run.smithi066.stdout: python3-trustme-0.6.0-4.el8.noarch 2023-11-17T05:16:25.683 INFO:teuthology.orchestra.run.smithi066.stdout: python3-waitress-1.4.3-1.el8.noarch 2023-11-17T05:16:25.683 INFO:teuthology.orchestra.run.smithi066.stdout: python3-webob-1.8.5-1.el8.1.noarch 2023-11-17T05:16:25.683 INFO:teuthology.orchestra.run.smithi066.stdout: python3-webtest-2.0.33-1.el8.noarch 2023-11-17T05:16:25.683 INFO:teuthology.orchestra.run.smithi066.stdout: python3-werkzeug-0.12.2-4.el8.noarch 2023-11-17T05:16:25.684 INFO:teuthology.orchestra.run.smithi066.stdout: python3-zc-lockfile-2.0-2.el8.noarch 2023-11-17T05:16:25.684 INFO:teuthology.orchestra.run.smithi066.stdout: 2023-11-17T05:16:25.684 INFO:teuthology.orchestra.run.smithi066.stdout:Complete! 2023-11-17T05:16:25.897 INFO:teuthology.orchestra.run.smithi040.stdout:(6/32): ceph-osd-16.2.4-0.el8.x86_64.rpm 43 MB/s | 18 MB 00:00 2023-11-17T05:16:25.914 INFO:teuthology.orchestra.run.smithi040.stdout:(7/32): python3-markupsafe-0.23-19.el8.x86_64.r 145 kB/s | 39 kB 00:00 2023-11-17T05:16:25.918 DEBUG:teuthology.orchestra.run.smithi066:> sudo yum -y install ceph-base 2023-11-17T05:16:25.939 INFO:teuthology.orchestra.run.smithi040.stdout:(8/32): python3-mako-1.0.6-14.el8.noarch.rpm 408 kB/s | 157 kB 00:00 2023-11-17T05:16:25.998 INFO:teuthology.orchestra.run.smithi040.stdout:(9/32): python3-pytz-2017.2-11.el8.noarch.rpm 682 kB/s | 57 kB 00:00 2023-11-17T05:16:26.115 INFO:teuthology.orchestra.run.smithi040.stdout:(10/32): python3-werkzeug-0.12.2-4.el8.noarch.r 2.6 MB/s | 457 kB 00:00 2023-11-17T05:16:26.132 INFO:teuthology.orchestra.run.smithi040.stdout:(11/32): python3-pyOpenSSL-19.0.0-1.el8.noarch. 441 kB/s | 103 kB 00:00 2023-11-17T05:16:26.157 INFO:teuthology.orchestra.run.smithi040.stdout:(12/32): python3-cffi-1.11.5-6.el8.x86_64.rpm 1.5 MB/s | 249 kB 00:00 2023-11-17T05:16:26.257 INFO:teuthology.orchestra.run.smithi040.stdout:(13/32): python3-pycparser-2.14-14.el8.noarch.r 871 kB/s | 109 kB 00:00 2023-11-17T05:16:26.283 INFO:teuthology.orchestra.run.smithi040.stdout:(14/32): python3-cryptography-3.2.1-6.el8.x86_6 3.7 MB/s | 641 kB 00:00 2023-11-17T05:16:26.399 INFO:teuthology.orchestra.run.smithi040.stdout:(15/32): python3-bcrypt-3.1.6-2.el8.1.x86_64.rp 180 kB/s | 44 kB 00:00 2023-11-17T05:16:26.486 INFO:teuthology.orchestra.run.smithi040.stdout:(16/32): python3-beautifulsoup4-4.6.3-2.el8.1.n 822 kB/s | 185 kB 00:00 2023-11-17T05:16:26.508 INFO:teuthology.orchestra.run.smithi040.stdout:(17/32): python3-cheroot-8.5.2-1.el8.noarch.rpm 766 kB/s | 173 kB 00:00 2023-11-17T05:16:26.558 INFO:teuthology.orchestra.run.smithi040.stdout:(18/32): python3-jaraco-6.2-6.el8.noarch.rpm 147 kB/s | 11 kB 00:00 2023-11-17T05:16:26.584 INFO:teuthology.orchestra.run.smithi040.stdout:(19/32): python3-jaraco-functools-2.0-4.el8.noa 234 kB/s | 18 kB 00:00 2023-11-17T05:16:26.634 INFO:teuthology.orchestra.run.smithi040.stdout:(20/32): python3-logutils-0.3.5-11.el8.noarch.r 647 kB/s | 49 kB 00:00 2023-11-17T05:16:26.676 INFO:teuthology.orchestra.run.smithi040.stdout:(21/32): python3-more-itertools-7.2.0-3.el8.noa 638 kB/s | 59 kB 00:00 2023-11-17T05:16:26.725 INFO:teuthology.orchestra.run.smithi040.stdout:(22/32): python3-cherrypy-18.4.0-1.el8.noarch.r 1.2 MB/s | 384 kB 00:00 2023-11-17T05:16:26.785 INFO:teuthology.orchestra.run.smithi040.stdout:(23/32): python3-portend-2.6-1.el8.noarch.rpm 144 kB/s | 16 kB 00:00 2023-11-17T05:16:26.810 INFO:teuthology.orchestra.run.smithi040.stdout:(24/32): python3-simplegeneric-0.8.1-17.el8.noa 220 kB/s | 19 kB 00:00 2023-11-17T05:16:26.813 INFO:teuthology.orchestra.run.smithi066.stdout:Last metadata expiration check: 0:01:13 ago on Fri 17 Nov 2023 05:15:13 AM UTC. 2023-11-17T05:16:26.836 INFO:teuthology.orchestra.run.smithi040.stdout:(25/32): python3-pecan-1.3.2-9.el8.noarch.rpm 1.4 MB/s | 283 kB 00:00 2023-11-17T05:16:26.861 INFO:teuthology.orchestra.run.smithi040.stdout:(26/32): python3-singledispatch-3.4.0.3-18.el8. 316 kB/s | 24 kB 00:00 2023-11-17T05:16:26.903 INFO:teuthology.orchestra.run.smithi040.stdout:(27/32): python3-tempora-1.14.1-5.el8.noarch.rp 317 kB/s | 29 kB 00:00 2023-11-17T05:16:26.928 INFO:teuthology.orchestra.run.smithi040.stdout:(28/32): python3-trustme-0.6.0-4.el8.noarch.rpm 298 kB/s | 27 kB 00:00 2023-11-17T05:16:26.987 INFO:teuthology.orchestra.run.smithi040.stdout:(29/32): python3-waitress-1.4.3-1.el8.noarch.rp 2.0 MB/s | 255 kB 00:00 2023-11-17T05:16:27.016 INFO:teuthology.orchestra.run.smithi040.stdout:(30/32): python3-webob-1.8.5-1.el8.1.noarch.rpm 2.3 MB/s | 251 kB 00:00 2023-11-17T05:16:27.037 INFO:teuthology.orchestra.run.smithi040.stdout:(31/32): python3-webtest-2.0.33-1.el8.noarch.rp 785 kB/s | 85 kB 00:00 2023-11-17T05:16:27.062 INFO:teuthology.orchestra.run.smithi040.stdout:(32/32): python3-zc-lockfile-2.0-2.el8.noarch.r 306 kB/s | 23 kB 00:00 2023-11-17T05:16:27.063 INFO:teuthology.orchestra.run.smithi040.stdout:-------------------------------------------------------------------------------- 2023-11-17T05:16:27.064 INFO:teuthology.orchestra.run.smithi040.stdout:Total 14 MB/s | 31 MB 00:02 2023-11-17T05:16:27.154 INFO:teuthology.orchestra.run.smithi040.stdout:Running transaction check 2023-11-17T05:16:27.197 INFO:teuthology.orchestra.run.smithi040.stdout:Transaction check succeeded. 2023-11-17T05:16:27.197 INFO:teuthology.orchestra.run.smithi040.stdout:Running transaction test 2023-11-17T05:16:27.550 INFO:teuthology.orchestra.run.smithi040.stdout:Transaction test succeeded. 2023-11-17T05:16:27.554 INFO:teuthology.orchestra.run.smithi040.stdout:Running transaction 2023-11-17T05:16:28.385 INFO:teuthology.orchestra.run.smithi066.stdout:Package ceph-base-2:16.2.4-0.el8.x86_64 is already installed. 2023-11-17T05:16:28.619 INFO:teuthology.orchestra.run.smithi066.stdout:Dependencies resolved. 2023-11-17T05:16:28.620 INFO:teuthology.orchestra.run.smithi066.stdout:Nothing to do. 2023-11-17T05:16:28.620 INFO:teuthology.orchestra.run.smithi066.stdout:Complete! 2023-11-17T05:16:28.689 INFO:teuthology.orchestra.run.smithi040.stdout: Preparing : 1/1 2023-11-17T05:16:28.737 DEBUG:teuthology.orchestra.run.smithi066:> sudo yum -y install cephadm 2023-11-17T05:16:28.891 INFO:teuthology.orchestra.run.smithi040.stdout: Installing : python3-more-itertools-7.2.0-3.el8.noarch 1/32 2023-11-17T05:16:29.077 INFO:teuthology.orchestra.run.smithi040.stdout: Installing : python3-webob-1.8.5-1.el8.1.noarch 2/32 2023-11-17T05:16:29.313 INFO:teuthology.orchestra.run.smithi040.stdout: Installing : python3-zc-lockfile-2.0-2.el8.noarch 3/32 2023-11-17T05:16:29.517 INFO:teuthology.orchestra.run.smithi040.stdout: Installing : python3-waitress-1.4.3-1.el8.noarch 4/32 2023-11-17T05:16:29.637 INFO:teuthology.orchestra.run.smithi066.stdout:Last metadata expiration check: 0:01:16 ago on Fri 17 Nov 2023 05:15:13 AM UTC. 2023-11-17T05:16:29.672 INFO:teuthology.orchestra.run.smithi040.stdout: Installing : python3-singledispatch-3.4.0.3-18.el8.noarch 5/32 2023-11-17T05:16:29.856 INFO:teuthology.orchestra.run.smithi040.stdout: Installing : python3-simplegeneric-0.8.1-17.el8.noarch 6/32 2023-11-17T05:16:30.037 INFO:teuthology.orchestra.run.smithi040.stdout: Installing : python3-logutils-0.3.5-11.el8.noarch 7/32 2023-11-17T05:16:30.232 INFO:teuthology.orchestra.run.smithi040.stdout: Installing : python3-jaraco-6.2-6.el8.noarch 8/32 2023-11-17T05:16:30.437 INFO:teuthology.orchestra.run.smithi040.stdout: Installing : python3-jaraco-functools-2.0-4.el8.noarch 9/32 2023-11-17T05:16:30.689 INFO:teuthology.orchestra.run.smithi040.stdout: Installing : python3-beautifulsoup4-4.6.3-2.el8.1.noarch 10/32 2023-11-17T05:16:30.913 INFO:teuthology.orchestra.run.smithi040.stdout: Installing : python3-webtest-2.0.33-1.el8.noarch 11/32 2023-11-17T05:16:31.136 INFO:teuthology.orchestra.run.smithi040.stdout: Installing : python3-pycparser-2.14-14.el8.noarch 12/32 2023-11-17T05:16:31.414 INFO:teuthology.orchestra.run.smithi066.stdout:Dependencies resolved. 2023-11-17T05:16:31.415 INFO:teuthology.orchestra.run.smithi066.stdout:================================================================================ 2023-11-17T05:16:31.415 INFO:teuthology.orchestra.run.smithi066.stdout: Package Architecture Version Repository Size 2023-11-17T05:16:31.415 INFO:teuthology.orchestra.run.smithi066.stdout:================================================================================ 2023-11-17T05:16:31.415 INFO:teuthology.orchestra.run.smithi066.stdout:Installing: 2023-11-17T05:16:31.415 INFO:teuthology.orchestra.run.smithi066.stdout: cephadm noarch 2:16.2.4-0.el8 ceph-noarch 71 k 2023-11-17T05:16:31.415 INFO:teuthology.orchestra.run.smithi066.stdout: 2023-11-17T05:16:31.416 INFO:teuthology.orchestra.run.smithi066.stdout:Transaction Summary 2023-11-17T05:16:31.416 INFO:teuthology.orchestra.run.smithi066.stdout:================================================================================ 2023-11-17T05:16:31.416 INFO:teuthology.orchestra.run.smithi066.stdout:Install 1 Package 2023-11-17T05:16:31.416 INFO:teuthology.orchestra.run.smithi066.stdout: 2023-11-17T05:16:31.416 INFO:teuthology.orchestra.run.smithi066.stdout:Total download size: 71 k 2023-11-17T05:16:31.416 INFO:teuthology.orchestra.run.smithi066.stdout:Installed size: 290 k 2023-11-17T05:16:31.416 INFO:teuthology.orchestra.run.smithi066.stdout:Downloading Packages: 2023-11-17T05:16:31.451 INFO:teuthology.orchestra.run.smithi066.stdout:cephadm-16.2.4-0.el8.noarch.rpm 2.0 MB/s | 71 kB 00:00 2023-11-17T05:16:31.452 INFO:teuthology.orchestra.run.smithi066.stdout:-------------------------------------------------------------------------------- 2023-11-17T05:16:31.452 INFO:teuthology.orchestra.run.smithi066.stdout:Total 1.9 MB/s | 71 kB 00:00 2023-11-17T05:16:31.452 INFO:teuthology.orchestra.run.smithi066.stdout:Running transaction check 2023-11-17T05:16:31.459 INFO:teuthology.orchestra.run.smithi066.stdout:Transaction check succeeded. 2023-11-17T05:16:31.460 INFO:teuthology.orchestra.run.smithi066.stdout:Running transaction test 2023-11-17T05:16:31.461 INFO:teuthology.orchestra.run.smithi066.stdout:Transaction test succeeded. 2023-11-17T05:16:31.465 INFO:teuthology.orchestra.run.smithi066.stdout:Running transaction 2023-11-17T05:16:31.504 INFO:teuthology.orchestra.run.smithi066.stdout: Preparing : 1/1 2023-11-17T05:16:31.575 INFO:teuthology.orchestra.run.smithi040.stdout: Installing : python3-cffi-1.11.5-6.el8.x86_64 13/32 2023-11-17T05:16:31.899 INFO:teuthology.orchestra.run.smithi040.stdout: Installing : python3-cryptography-3.2.1-6.el8.x86_64 14/32 2023-11-17T05:16:31.966 INFO:teuthology.orchestra.run.smithi066.stdout: Running scriptlet: cephadm-2:16.2.4-0.el8.noarch 1/1 2023-11-17T05:16:32.052 INFO:teuthology.orchestra.run.smithi040.stdout: Installing : python3-pyOpenSSL-19.0.0-1.el8.noarch 15/32 2023-11-17T05:16:32.172 INFO:teuthology.orchestra.run.smithi066.stdout: Installing : cephadm-2:16.2.4-0.el8.noarch 1/1 2023-11-17T05:16:32.309 INFO:teuthology.orchestra.run.smithi040.stdout: Installing : python3-trustme-0.6.0-4.el8.noarch 16/32 2023-11-17T05:16:32.408 INFO:teuthology.orchestra.run.smithi066.stdout: Running scriptlet: cephadm-2:16.2.4-0.el8.noarch 1/1 2023-11-17T05:16:32.537 INFO:teuthology.orchestra.run.smithi040.stdout: Installing : python3-cheroot-8.5.2-1.el8.noarch 17/32 2023-11-17T05:16:32.847 INFO:teuthology.orchestra.run.smithi040.stdout: Installing : python3-bcrypt-3.1.6-2.el8.1.x86_64 18/32 2023-11-17T05:16:33.042 INFO:teuthology.orchestra.run.smithi040.stdout: Installing : python3-werkzeug-0.12.2-4.el8.noarch 19/32 2023-11-17T05:16:53.168 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : cephadm-2:16.2.4-0.el8.noarch 1/1 2023-11-17T05:16:53.168 INFO:teuthology.orchestra.run.smithi066.stdout: 2023-11-17T05:16:53.168 INFO:teuthology.orchestra.run.smithi066.stdout:Installed: 2023-11-17T05:16:53.168 INFO:teuthology.orchestra.run.smithi066.stdout: cephadm-2:16.2.4-0.el8.noarch 2023-11-17T05:16:53.168 INFO:teuthology.orchestra.run.smithi066.stdout: 2023-11-17T05:16:53.168 INFO:teuthology.orchestra.run.smithi066.stdout:Complete! 2023-11-17T05:16:53.169 DEBUG:teuthology.orchestra.run.smithi066:> sudo yum -y install ceph-immutable-object-cache 2023-11-17T05:16:53.173 INFO:teuthology.orchestra.run.smithi040.stdout: Installing : python3-pytz-2017.2-11.el8.noarch 20/32 2023-11-17T05:16:53.233 INFO:teuthology.orchestra.run.smithi040.stdout: Installing : python3-tempora-1.14.1-5.el8.noarch 21/32 2023-11-17T05:16:53.234 INFO:teuthology.orchestra.run.smithi040.stdout: Installing : python3-portend-2.6-1.el8.noarch 22/32 2023-11-17T05:16:53.234 INFO:teuthology.orchestra.run.smithi040.stdout: Installing : python3-cherrypy-18.4.0-1.el8.noarch 23/32 2023-11-17T05:16:53.234 INFO:teuthology.orchestra.run.smithi040.stdout: Installing : python3-markupsafe-0.23-19.el8.x86_64 24/32 2023-11-17T05:16:53.234 INFO:teuthology.orchestra.run.smithi040.stdout: Installing : python3-mako-1.0.6-14.el8.noarch 25/32 2023-11-17T05:16:53.234 INFO:teuthology.orchestra.run.smithi040.stdout: Installing : python3-pecan-1.3.2-9.el8.noarch 26/32 2023-11-17T05:16:53.235 INFO:teuthology.orchestra.run.smithi040.stdout: Installing : ceph-mgr-modules-core-2:16.2.4-0.el8.noarch 27/32 2023-11-17T05:16:53.235 INFO:teuthology.orchestra.run.smithi040.stdout: Installing : ceph-mgr-2:16.2.4-0.el8.x86_64 28/32 2023-11-17T05:16:53.235 INFO:teuthology.orchestra.run.smithi040.stdout: Running scriptlet: ceph-mgr-2:16.2.4-0.el8.x86_64 28/32 2023-11-17T05:16:53.235 INFO:teuthology.orchestra.run.smithi040.stdout: Installing : ceph-osd-2:16.2.4-0.el8.x86_64 29/32 2023-11-17T05:16:53.235 INFO:teuthology.orchestra.run.smithi040.stdout: Running scriptlet: ceph-osd-2:16.2.4-0.el8.x86_64 29/32 2023-11-17T05:16:53.236 INFO:teuthology.orchestra.run.smithi040.stdout: Installing : ceph-mon-2:16.2.4-0.el8.x86_64 30/32 2023-11-17T05:16:53.236 INFO:teuthology.orchestra.run.smithi040.stdout: Running scriptlet: ceph-mon-2:16.2.4-0.el8.x86_64 30/32 2023-11-17T05:16:53.236 INFO:teuthology.orchestra.run.smithi040.stdout: Installing : ceph-mds-2:16.2.4-0.el8.x86_64 31/32 2023-11-17T05:16:53.236 INFO:teuthology.orchestra.run.smithi040.stdout: Running scriptlet: ceph-mds-2:16.2.4-0.el8.x86_64 31/32 2023-11-17T05:16:53.236 INFO:teuthology.orchestra.run.smithi040.stdout: Installing : ceph-2:16.2.4-0.el8.x86_64 32/32 2023-11-17T05:16:53.237 INFO:teuthology.orchestra.run.smithi040.stdout: Running scriptlet: ceph-2:16.2.4-0.el8.x86_64 32/32 2023-11-17T05:16:53.237 INFO:teuthology.orchestra.run.smithi040.stdout: Verifying : ceph-2:16.2.4-0.el8.x86_64 1/32 2023-11-17T05:16:53.237 INFO:teuthology.orchestra.run.smithi040.stdout: Verifying : ceph-mds-2:16.2.4-0.el8.x86_64 2/32 2023-11-17T05:16:53.237 INFO:teuthology.orchestra.run.smithi040.stdout: Verifying : ceph-mgr-2:16.2.4-0.el8.x86_64 3/32 2023-11-17T05:16:53.237 INFO:teuthology.orchestra.run.smithi040.stdout: Verifying : ceph-mon-2:16.2.4-0.el8.x86_64 4/32 2023-11-17T05:16:53.237 INFO:teuthology.orchestra.run.smithi040.stdout: Verifying : ceph-osd-2:16.2.4-0.el8.x86_64 5/32 2023-11-17T05:16:53.238 INFO:teuthology.orchestra.run.smithi040.stdout: Verifying : ceph-mgr-modules-core-2:16.2.4-0.el8.noarch 6/32 2023-11-17T05:16:53.241 INFO:teuthology.orchestra.run.smithi040.stdout: Verifying : python3-mako-1.0.6-14.el8.noarch 7/32 2023-11-17T05:16:53.286 INFO:teuthology.orchestra.run.smithi040.stdout: Verifying : python3-markupsafe-0.23-19.el8.x86_64 8/32 2023-11-17T05:16:53.286 INFO:teuthology.orchestra.run.smithi040.stdout: Verifying : python3-pyOpenSSL-19.0.0-1.el8.noarch 9/32 2023-11-17T05:16:53.287 INFO:teuthology.orchestra.run.smithi040.stdout: Verifying : python3-pytz-2017.2-11.el8.noarch 10/32 2023-11-17T05:16:53.287 INFO:teuthology.orchestra.run.smithi040.stdout: Verifying : python3-werkzeug-0.12.2-4.el8.noarch 11/32 2023-11-17T05:16:53.287 INFO:teuthology.orchestra.run.smithi040.stdout: Verifying : python3-cffi-1.11.5-6.el8.x86_64 12/32 2023-11-17T05:16:53.287 INFO:teuthology.orchestra.run.smithi040.stdout: Verifying : python3-cryptography-3.2.1-6.el8.x86_64 13/32 2023-11-17T05:16:53.287 INFO:teuthology.orchestra.run.smithi040.stdout: Verifying : python3-pycparser-2.14-14.el8.noarch 14/32 2023-11-17T05:16:53.287 INFO:teuthology.orchestra.run.smithi040.stdout: Verifying : python3-bcrypt-3.1.6-2.el8.1.x86_64 15/32 2023-11-17T05:16:53.287 INFO:teuthology.orchestra.run.smithi040.stdout: Verifying : python3-beautifulsoup4-4.6.3-2.el8.1.noarch 16/32 2023-11-17T05:16:53.287 INFO:teuthology.orchestra.run.smithi040.stdout: Verifying : python3-cheroot-8.5.2-1.el8.noarch 17/32 2023-11-17T05:16:53.287 INFO:teuthology.orchestra.run.smithi040.stdout: Verifying : python3-cherrypy-18.4.0-1.el8.noarch 18/32 2023-11-17T05:16:53.287 INFO:teuthology.orchestra.run.smithi040.stdout: Verifying : python3-jaraco-6.2-6.el8.noarch 19/32 2023-11-17T05:16:53.287 INFO:teuthology.orchestra.run.smithi040.stdout: Verifying : python3-jaraco-functools-2.0-4.el8.noarch 20/32 2023-11-17T05:16:53.287 INFO:teuthology.orchestra.run.smithi040.stdout: Verifying : python3-logutils-0.3.5-11.el8.noarch 21/32 2023-11-17T05:16:53.287 INFO:teuthology.orchestra.run.smithi040.stdout: Verifying : python3-more-itertools-7.2.0-3.el8.noarch 22/32 2023-11-17T05:16:53.288 INFO:teuthology.orchestra.run.smithi040.stdout: Verifying : python3-pecan-1.3.2-9.el8.noarch 23/32 2023-11-17T05:16:53.288 INFO:teuthology.orchestra.run.smithi040.stdout: Verifying : python3-portend-2.6-1.el8.noarch 24/32 2023-11-17T05:16:53.288 INFO:teuthology.orchestra.run.smithi040.stdout: Verifying : python3-simplegeneric-0.8.1-17.el8.noarch 25/32 2023-11-17T05:16:53.288 INFO:teuthology.orchestra.run.smithi040.stdout: Verifying : python3-singledispatch-3.4.0.3-18.el8.noarch 26/32 2023-11-17T05:16:53.288 INFO:teuthology.orchestra.run.smithi040.stdout: Verifying : python3-tempora-1.14.1-5.el8.noarch 27/32 2023-11-17T05:16:53.288 INFO:teuthology.orchestra.run.smithi040.stdout: Verifying : python3-trustme-0.6.0-4.el8.noarch 28/32 2023-11-17T05:16:53.288 INFO:teuthology.orchestra.run.smithi040.stdout: Verifying : python3-waitress-1.4.3-1.el8.noarch 29/32 2023-11-17T05:16:53.288 INFO:teuthology.orchestra.run.smithi040.stdout: Verifying : python3-webob-1.8.5-1.el8.1.noarch 30/32 2023-11-17T05:16:53.288 INFO:teuthology.orchestra.run.smithi040.stdout: Verifying : python3-webtest-2.0.33-1.el8.noarch 31/32 2023-11-17T05:16:53.288 INFO:teuthology.orchestra.run.smithi040.stdout: Verifying : python3-zc-lockfile-2.0-2.el8.noarch 32/32 2023-11-17T05:16:53.288 INFO:teuthology.orchestra.run.smithi040.stdout: 2023-11-17T05:16:53.288 INFO:teuthology.orchestra.run.smithi040.stdout:Installed: 2023-11-17T05:16:53.288 INFO:teuthology.orchestra.run.smithi040.stdout: ceph-2:16.2.4-0.el8.x86_64 2023-11-17T05:16:53.289 INFO:teuthology.orchestra.run.smithi040.stdout: ceph-mds-2:16.2.4-0.el8.x86_64 2023-11-17T05:16:53.289 INFO:teuthology.orchestra.run.smithi040.stdout: ceph-mgr-2:16.2.4-0.el8.x86_64 2023-11-17T05:16:53.289 INFO:teuthology.orchestra.run.smithi040.stdout: ceph-mgr-modules-core-2:16.2.4-0.el8.noarch 2023-11-17T05:16:53.289 INFO:teuthology.orchestra.run.smithi040.stdout: ceph-mon-2:16.2.4-0.el8.x86_64 2023-11-17T05:16:53.289 INFO:teuthology.orchestra.run.smithi040.stdout: ceph-osd-2:16.2.4-0.el8.x86_64 2023-11-17T05:16:53.289 INFO:teuthology.orchestra.run.smithi040.stdout: python3-bcrypt-3.1.6-2.el8.1.x86_64 2023-11-17T05:16:53.289 INFO:teuthology.orchestra.run.smithi040.stdout: python3-beautifulsoup4-4.6.3-2.el8.1.noarch 2023-11-17T05:16:53.289 INFO:teuthology.orchestra.run.smithi040.stdout: python3-cffi-1.11.5-6.el8.x86_64 2023-11-17T05:16:53.289 INFO:teuthology.orchestra.run.smithi040.stdout: python3-cheroot-8.5.2-1.el8.noarch 2023-11-17T05:16:53.289 INFO:teuthology.orchestra.run.smithi040.stdout: python3-cherrypy-18.4.0-1.el8.noarch 2023-11-17T05:16:53.289 INFO:teuthology.orchestra.run.smithi040.stdout: python3-cryptography-3.2.1-6.el8.x86_64 2023-11-17T05:16:53.289 INFO:teuthology.orchestra.run.smithi040.stdout: python3-jaraco-6.2-6.el8.noarch 2023-11-17T05:16:53.289 INFO:teuthology.orchestra.run.smithi040.stdout: python3-jaraco-functools-2.0-4.el8.noarch 2023-11-17T05:16:53.290 INFO:teuthology.orchestra.run.smithi040.stdout: python3-logutils-0.3.5-11.el8.noarch 2023-11-17T05:16:53.290 INFO:teuthology.orchestra.run.smithi040.stdout: python3-mako-1.0.6-14.el8.noarch 2023-11-17T05:16:53.290 INFO:teuthology.orchestra.run.smithi040.stdout: python3-markupsafe-0.23-19.el8.x86_64 2023-11-17T05:16:53.290 INFO:teuthology.orchestra.run.smithi040.stdout: python3-more-itertools-7.2.0-3.el8.noarch 2023-11-17T05:16:53.290 INFO:teuthology.orchestra.run.smithi040.stdout: python3-pecan-1.3.2-9.el8.noarch 2023-11-17T05:16:53.290 INFO:teuthology.orchestra.run.smithi040.stdout: python3-portend-2.6-1.el8.noarch 2023-11-17T05:16:53.379 INFO:teuthology.orchestra.run.smithi040.stdout: python3-pyOpenSSL-19.0.0-1.el8.noarch 2023-11-17T05:16:53.379 INFO:teuthology.orchestra.run.smithi040.stdout: python3-pycparser-2.14-14.el8.noarch 2023-11-17T05:16:53.379 INFO:teuthology.orchestra.run.smithi040.stdout: python3-pytz-2017.2-11.el8.noarch 2023-11-17T05:16:53.380 INFO:teuthology.orchestra.run.smithi040.stdout: python3-simplegeneric-0.8.1-17.el8.noarch 2023-11-17T05:16:53.380 INFO:teuthology.orchestra.run.smithi040.stdout: python3-singledispatch-3.4.0.3-18.el8.noarch 2023-11-17T05:16:53.380 INFO:teuthology.orchestra.run.smithi040.stdout: python3-tempora-1.14.1-5.el8.noarch 2023-11-17T05:16:53.380 INFO:teuthology.orchestra.run.smithi040.stdout: python3-trustme-0.6.0-4.el8.noarch 2023-11-17T05:16:53.380 INFO:teuthology.orchestra.run.smithi040.stdout: python3-waitress-1.4.3-1.el8.noarch 2023-11-17T05:16:53.380 INFO:teuthology.orchestra.run.smithi040.stdout: python3-webob-1.8.5-1.el8.1.noarch 2023-11-17T05:16:53.380 INFO:teuthology.orchestra.run.smithi040.stdout: python3-webtest-2.0.33-1.el8.noarch 2023-11-17T05:16:53.380 INFO:teuthology.orchestra.run.smithi040.stdout: python3-werkzeug-0.12.2-4.el8.noarch 2023-11-17T05:16:53.380 INFO:teuthology.orchestra.run.smithi040.stdout: python3-zc-lockfile-2.0-2.el8.noarch 2023-11-17T05:16:53.380 INFO:teuthology.orchestra.run.smithi040.stdout: 2023-11-17T05:16:53.380 INFO:teuthology.orchestra.run.smithi040.stdout:Complete! 2023-11-17T05:16:53.381 DEBUG:teuthology.orchestra.run.smithi040:> sudo yum -y install ceph-base 2023-11-17T05:16:54.078 INFO:teuthology.orchestra.run.smithi066.stdout:Last metadata expiration check: 0:01:41 ago on Fri 17 Nov 2023 05:15:13 AM UTC. 2023-11-17T05:16:54.380 INFO:teuthology.orchestra.run.smithi040.stdout:Last metadata expiration check: 0:01:36 ago on Fri 17 Nov 2023 05:15:18 AM UTC. 2023-11-17T05:16:55.850 INFO:teuthology.orchestra.run.smithi066.stdout:Dependencies resolved. 2023-11-17T05:18:02.712 INFO:teuthology.orchestra.run.smithi040.stdout:Package ceph-base-2:16.2.4-0.el8.x86_64 is already installed. 2023-11-17T05:18:02.729 INFO:teuthology.orchestra.run.smithi040.stdout:Dependencies resolved. 2023-11-17T05:18:02.760 INFO:teuthology.orchestra.run.smithi040.stdout:Nothing to do. 2023-11-17T05:18:02.914 INFO:teuthology.orchestra.run.smithi040.stdout:Complete! 2023-11-17T05:18:02.915 DEBUG:teuthology.orchestra.run.smithi040:> sudo yum -y install cephadm 2023-11-17T05:18:02.917 INFO:teuthology.orchestra.run.smithi066.stdout:================================================================================ 2023-11-17T05:18:02.917 INFO:teuthology.orchestra.run.smithi066.stdout: Package Arch Version Repo Size 2023-11-17T05:18:02.978 INFO:teuthology.orchestra.run.smithi066.stdout:================================================================================ 2023-11-17T05:18:03.051 INFO:teuthology.orchestra.run.smithi066.stdout:Installing: 2023-11-17T05:18:03.051 INFO:teuthology.orchestra.run.smithi066.stdout: ceph-immutable-object-cache x86_64 2:16.2.4-0.el8 ceph 163 k 2023-11-17T05:18:03.051 INFO:teuthology.orchestra.run.smithi066.stdout: 2023-11-17T05:18:03.051 INFO:teuthology.orchestra.run.smithi066.stdout:Transaction Summary 2023-11-17T05:18:03.051 INFO:teuthology.orchestra.run.smithi066.stdout:================================================================================ 2023-11-17T05:18:03.051 INFO:teuthology.orchestra.run.smithi066.stdout:Install 1 Package 2023-11-17T05:18:03.051 INFO:teuthology.orchestra.run.smithi066.stdout: 2023-11-17T05:18:03.051 INFO:teuthology.orchestra.run.smithi066.stdout:Total download size: 163 k 2023-11-17T05:18:03.051 INFO:teuthology.orchestra.run.smithi066.stdout:Installed size: 587 k 2023-11-17T05:18:03.052 INFO:teuthology.orchestra.run.smithi066.stdout:Downloading Packages: 2023-11-17T05:18:03.052 INFO:teuthology.orchestra.run.smithi066.stdout:ceph-immutable-object-cache-16.2.4-0.el8.x86_64 2.9 MB/s | 163 kB 00:00 2023-11-17T05:18:03.052 INFO:teuthology.orchestra.run.smithi066.stdout:-------------------------------------------------------------------------------- 2023-11-17T05:18:03.052 INFO:teuthology.orchestra.run.smithi066.stdout:Total 2.8 MB/s | 163 kB 00:00 2023-11-17T05:18:03.052 INFO:teuthology.orchestra.run.smithi066.stdout:Running transaction check 2023-11-17T05:18:03.052 INFO:teuthology.orchestra.run.smithi066.stdout:Transaction check succeeded. 2023-11-17T05:18:03.052 INFO:teuthology.orchestra.run.smithi066.stdout:Running transaction test 2023-11-17T05:18:03.052 INFO:teuthology.orchestra.run.smithi066.stdout:Transaction test succeeded. 2023-11-17T05:18:03.052 INFO:teuthology.orchestra.run.smithi066.stdout:Running transaction 2023-11-17T05:18:03.052 INFO:teuthology.orchestra.run.smithi066.stdout: Preparing : 1/1 2023-11-17T05:18:03.052 INFO:teuthology.orchestra.run.smithi066.stdout: Installing : ceph-immutable-object-cache-2:16.2.4-0.el8.x86_64 1/1 2023-11-17T05:18:03.052 INFO:teuthology.orchestra.run.smithi066.stdout: Running scriptlet: ceph-immutable-object-cache-2:16.2.4-0.el8.x86_64 1/1 2023-11-17T05:18:03.052 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : ceph-immutable-object-cache-2:16.2.4-0.el8.x86_64 1/1 2023-11-17T05:18:03.053 INFO:teuthology.orchestra.run.smithi066.stdout: 2023-11-17T05:18:03.053 INFO:teuthology.orchestra.run.smithi066.stdout:Installed: 2023-11-17T05:18:03.053 INFO:teuthology.orchestra.run.smithi066.stdout: ceph-immutable-object-cache-2:16.2.4-0.el8.x86_64 2023-11-17T05:18:03.053 INFO:teuthology.orchestra.run.smithi066.stdout: 2023-11-17T05:18:03.053 INFO:teuthology.orchestra.run.smithi066.stdout:Complete! 2023-11-17T05:18:03.053 DEBUG:teuthology.orchestra.run.smithi066:> sudo yum -y install ceph-mgr 2023-11-17T05:18:03.915 INFO:teuthology.orchestra.run.smithi040.stdout:Last metadata expiration check: 0:02:45 ago on Fri 17 Nov 2023 05:15:18 AM UTC. 2023-11-17T05:18:03.967 INFO:teuthology.orchestra.run.smithi066.stdout:Last metadata expiration check: 0:02:50 ago on Fri 17 Nov 2023 05:15:13 AM UTC. 2023-11-17T05:18:05.511 INFO:teuthology.orchestra.run.smithi066.stdout:Package ceph-mgr-2:16.2.4-0.el8.x86_64 is already installed. 2023-11-17T05:18:05.745 INFO:teuthology.orchestra.run.smithi066.stdout:Dependencies resolved. 2023-11-17T05:18:05.746 INFO:teuthology.orchestra.run.smithi066.stdout:Nothing to do. 2023-11-17T05:18:05.746 INFO:teuthology.orchestra.run.smithi066.stdout:Complete! 2023-11-17T05:18:05.796 INFO:teuthology.orchestra.run.smithi040.stdout:Dependencies resolved. 2023-11-17T05:18:05.796 INFO:teuthology.orchestra.run.smithi040.stdout:================================================================================ 2023-11-17T05:18:05.797 INFO:teuthology.orchestra.run.smithi040.stdout: Package Architecture Version Repository Size 2023-11-17T05:18:05.797 INFO:teuthology.orchestra.run.smithi040.stdout:================================================================================ 2023-11-17T05:18:05.797 INFO:teuthology.orchestra.run.smithi040.stdout:Installing: 2023-11-17T05:18:05.797 INFO:teuthology.orchestra.run.smithi040.stdout: cephadm noarch 2:16.2.4-0.el8 ceph-noarch 71 k 2023-11-17T05:18:05.797 INFO:teuthology.orchestra.run.smithi040.stdout: 2023-11-17T05:18:05.797 INFO:teuthology.orchestra.run.smithi040.stdout:Transaction Summary 2023-11-17T05:18:05.797 INFO:teuthology.orchestra.run.smithi040.stdout:================================================================================ 2023-11-17T05:18:05.797 INFO:teuthology.orchestra.run.smithi040.stdout:Install 1 Package 2023-11-17T05:18:05.797 INFO:teuthology.orchestra.run.smithi040.stdout: 2023-11-17T05:18:05.798 INFO:teuthology.orchestra.run.smithi040.stdout:Total download size: 71 k 2023-11-17T05:18:05.798 INFO:teuthology.orchestra.run.smithi040.stdout:Installed size: 290 k 2023-11-17T05:18:05.798 INFO:teuthology.orchestra.run.smithi040.stdout:Downloading Packages: 2023-11-17T05:18:05.847 INFO:teuthology.orchestra.run.smithi040.stdout:cephadm-16.2.4-0.el8.noarch.rpm 1.4 MB/s | 71 kB 00:00 2023-11-17T05:18:05.847 INFO:teuthology.orchestra.run.smithi040.stdout:-------------------------------------------------------------------------------- 2023-11-17T05:18:05.848 INFO:teuthology.orchestra.run.smithi040.stdout:Total 1.4 MB/s | 71 kB 00:00 2023-11-17T05:18:05.848 INFO:teuthology.orchestra.run.smithi040.stdout:Running transaction check 2023-11-17T05:18:05.856 INFO:teuthology.orchestra.run.smithi040.stdout:Transaction check succeeded. 2023-11-17T05:18:05.856 INFO:teuthology.orchestra.run.smithi040.stdout:Running transaction test 2023-11-17T05:18:05.858 INFO:teuthology.orchestra.run.smithi040.stdout:Transaction test succeeded. 2023-11-17T05:18:05.860 DEBUG:teuthology.orchestra.run.smithi066:> sudo yum -y install ceph-mgr-dashboard 2023-11-17T05:18:05.861 INFO:teuthology.orchestra.run.smithi040.stdout:Running transaction 2023-11-17T05:18:05.905 INFO:teuthology.orchestra.run.smithi040.stdout: Preparing : 1/1 2023-11-17T05:18:06.501 INFO:teuthology.orchestra.run.smithi040.stdout: Running scriptlet: cephadm-2:16.2.4-0.el8.noarch 1/1 2023-11-17T05:18:06.686 INFO:teuthology.orchestra.run.smithi040.stdout: Installing : cephadm-2:16.2.4-0.el8.noarch 1/1 2023-11-17T05:18:06.746 INFO:teuthology.orchestra.run.smithi066.stdout:Last metadata expiration check: 0:02:53 ago on Fri 17 Nov 2023 05:15:13 AM UTC. 2023-11-17T05:18:06.946 INFO:teuthology.orchestra.run.smithi040.stdout: Running scriptlet: cephadm-2:16.2.4-0.el8.noarch 1/1 2023-11-17T05:18:07.659 INFO:teuthology.orchestra.run.smithi040.stdout: Verifying : cephadm-2:16.2.4-0.el8.noarch 1/1 2023-11-17T05:18:07.660 INFO:teuthology.orchestra.run.smithi040.stdout: 2023-11-17T05:18:07.660 INFO:teuthology.orchestra.run.smithi040.stdout:Installed: 2023-11-17T05:18:07.660 INFO:teuthology.orchestra.run.smithi040.stdout: cephadm-2:16.2.4-0.el8.noarch 2023-11-17T05:18:07.660 INFO:teuthology.orchestra.run.smithi040.stdout: 2023-11-17T05:18:07.660 INFO:teuthology.orchestra.run.smithi040.stdout:Complete! 2023-11-17T05:18:07.924 DEBUG:teuthology.orchestra.run.smithi040:> sudo yum -y install ceph-immutable-object-cache 2023-11-17T05:18:08.550 INFO:teuthology.orchestra.run.smithi066.stdout:Dependencies resolved. 2023-11-17T05:18:08.551 INFO:teuthology.orchestra.run.smithi066.stdout:================================================================================ 2023-11-17T05:18:08.551 INFO:teuthology.orchestra.run.smithi066.stdout: Package Arch Version Repository Size 2023-11-17T05:18:08.551 INFO:teuthology.orchestra.run.smithi066.stdout:================================================================================ 2023-11-17T05:18:08.551 INFO:teuthology.orchestra.run.smithi066.stdout:Installing: 2023-11-17T05:18:08.551 INFO:teuthology.orchestra.run.smithi066.stdout: ceph-mgr-dashboard noarch 2:16.2.4-0.el8 ceph-noarch 2.7 M 2023-11-17T05:18:08.552 INFO:teuthology.orchestra.run.smithi066.stdout:Installing dependencies: 2023-11-17T05:18:08.552 INFO:teuthology.orchestra.run.smithi066.stdout: ceph-grafana-dashboards noarch 2:16.2.4-0.el8 ceph-noarch 24 k 2023-11-17T05:18:08.552 INFO:teuthology.orchestra.run.smithi066.stdout: ceph-prometheus-alerts noarch 2:16.2.4-0.el8 ceph-noarch 8.9 k 2023-11-17T05:18:08.552 INFO:teuthology.orchestra.run.smithi066.stdout: python3-jwt noarch 1.6.1-2.el8 baseos 43 k 2023-11-17T05:18:08.552 INFO:teuthology.orchestra.run.smithi066.stdout: python3-repoze-lru noarch 0.7-6.el8 epel 34 k 2023-11-17T05:18:08.552 INFO:teuthology.orchestra.run.smithi066.stdout: python3-routes noarch 2.4.1-12.el8 epel 196 k 2023-11-17T05:18:08.552 INFO:teuthology.orchestra.run.smithi066.stdout: 2023-11-17T05:18:08.552 INFO:teuthology.orchestra.run.smithi066.stdout:Transaction Summary 2023-11-17T05:18:08.552 INFO:teuthology.orchestra.run.smithi066.stdout:================================================================================ 2023-11-17T05:18:08.552 INFO:teuthology.orchestra.run.smithi066.stdout:Install 6 Packages 2023-11-17T05:18:08.552 INFO:teuthology.orchestra.run.smithi066.stdout: 2023-11-17T05:18:08.553 INFO:teuthology.orchestra.run.smithi066.stdout:Total download size: 3.0 M 2023-11-17T05:18:08.553 INFO:teuthology.orchestra.run.smithi066.stdout:Installed size: 15 M 2023-11-17T05:18:08.553 INFO:teuthology.orchestra.run.smithi066.stdout:Downloading Packages: 2023-11-17T05:18:08.911 INFO:teuthology.orchestra.run.smithi040.stdout:Last metadata expiration check: 0:02:50 ago on Fri 17 Nov 2023 05:15:18 AM UTC. 2023-11-17T05:18:09.061 INFO:teuthology.orchestra.run.smithi066.stdout:(1/6): ceph-grafana-dashboards-16.2.4-0.el8.noa 449 kB/s | 24 kB 00:00 2023-11-17T05:18:11.778 INFO:teuthology.orchestra.run.smithi040.stdout:Dependencies resolved. 2023-11-17T05:18:11.778 INFO:teuthology.orchestra.run.smithi040.stdout:================================================================================ 2023-11-17T05:18:11.778 INFO:teuthology.orchestra.run.smithi040.stdout: Package Arch Version Repo Size 2023-11-17T05:18:11.778 INFO:teuthology.orchestra.run.smithi040.stdout:================================================================================ 2023-11-17T05:18:11.778 INFO:teuthology.orchestra.run.smithi040.stdout:Installing: 2023-11-17T05:18:11.779 INFO:teuthology.orchestra.run.smithi040.stdout: ceph-immutable-object-cache x86_64 2:16.2.4-0.el8 ceph 163 k 2023-11-17T05:18:11.779 INFO:teuthology.orchestra.run.smithi040.stdout: 2023-11-17T05:18:11.779 INFO:teuthology.orchestra.run.smithi040.stdout:Transaction Summary 2023-11-17T05:18:11.779 INFO:teuthology.orchestra.run.smithi040.stdout:================================================================================ 2023-11-17T05:18:11.779 INFO:teuthology.orchestra.run.smithi040.stdout:Install 1 Package 2023-11-17T05:18:11.780 INFO:teuthology.orchestra.run.smithi040.stdout: 2023-11-17T05:18:11.780 INFO:teuthology.orchestra.run.smithi040.stdout:Total download size: 163 k 2023-11-17T05:18:11.780 INFO:teuthology.orchestra.run.smithi040.stdout:Installed size: 587 k 2023-11-17T05:18:11.780 INFO:teuthology.orchestra.run.smithi040.stdout:Downloading Packages: 2023-11-17T05:18:11.780 INFO:teuthology.orchestra.run.smithi040.stdout:ceph-immutable-object-cache-16.2.4-0.el8.x86_64 3.1 MB/s | 163 kB 00:00 2023-11-17T05:18:11.781 INFO:teuthology.orchestra.run.smithi040.stdout:-------------------------------------------------------------------------------- 2023-11-17T05:18:11.793 INFO:teuthology.orchestra.run.smithi040.stdout:Total 3.0 MB/s | 163 kB 00:00 2023-11-17T05:18:11.793 INFO:teuthology.orchestra.run.smithi040.stdout:Running transaction check 2023-11-17T05:18:11.793 INFO:teuthology.orchestra.run.smithi040.stdout:Transaction check succeeded. 2023-11-17T05:18:11.793 INFO:teuthology.orchestra.run.smithi040.stdout:Running transaction test 2023-11-17T05:18:11.794 INFO:teuthology.orchestra.run.smithi040.stdout:Transaction test succeeded. 2023-11-17T05:18:11.794 INFO:teuthology.orchestra.run.smithi040.stdout:Running transaction 2023-11-17T05:18:11.794 INFO:teuthology.orchestra.run.smithi040.stdout: Preparing : 1/1 2023-11-17T05:18:11.794 INFO:teuthology.orchestra.run.smithi040.stdout: Installing : ceph-immutable-object-cache-2:16.2.4-0.el8.x86_64 1/1 2023-11-17T05:18:11.794 INFO:teuthology.orchestra.run.smithi040.stdout: Running scriptlet: ceph-immutable-object-cache-2:16.2.4-0.el8.x86_64 1/1 2023-11-17T05:18:11.795 INFO:teuthology.orchestra.run.smithi066.stdout:(2/6): ceph-prometheus-alerts-16.2.4-0.el8.noar 114 kB/s | 8.9 kB 00:00 2023-11-17T05:18:11.795 INFO:teuthology.orchestra.run.smithi066.stdout:(3/6): ceph-mgr-dashboard-16.2.4-0.el8.noarch.r 18 MB/s | 2.7 MB 00:00 2023-11-17T05:18:11.795 INFO:teuthology.orchestra.run.smithi066.stdout:(4/6): python3-jwt-1.6.1-2.el8.noarch.rpm 212 kB/s | 43 kB 00:00 2023-11-17T05:18:11.796 INFO:teuthology.orchestra.run.smithi066.stdout:(5/6): python3-repoze-lru-0.7-6.el8.noarch.rpm 76 kB/s | 34 kB 00:00 2023-11-17T05:18:11.796 INFO:teuthology.orchestra.run.smithi066.stdout:(6/6): python3-routes-2.4.1-12.el8.noarch.rpm 491 kB/s | 196 kB 00:00 2023-11-17T05:18:11.796 INFO:teuthology.orchestra.run.smithi066.stdout:-------------------------------------------------------------------------------- 2023-11-17T05:18:11.796 INFO:teuthology.orchestra.run.smithi066.stdout:Total 2.9 MB/s | 3.0 MB 00:01 2023-11-17T05:18:11.796 INFO:teuthology.orchestra.run.smithi066.stdout:Running transaction check 2023-11-17T05:18:11.796 INFO:teuthology.orchestra.run.smithi066.stdout:Transaction check succeeded. 2023-11-17T05:18:11.797 INFO:teuthology.orchestra.run.smithi066.stdout:Running transaction test 2023-11-17T05:18:11.797 INFO:teuthology.orchestra.run.smithi066.stdout:Transaction test succeeded. 2023-11-17T05:18:11.797 INFO:teuthology.orchestra.run.smithi066.stdout:Running transaction 2023-11-17T05:18:11.797 INFO:teuthology.orchestra.run.smithi066.stdout: Preparing : 1/1 2023-11-17T05:18:11.797 INFO:teuthology.orchestra.run.smithi066.stdout: Installing : python3-repoze-lru-0.7-6.el8.noarch 1/6 2023-11-17T05:18:11.798 INFO:teuthology.orchestra.run.smithi066.stdout: Installing : python3-routes-2.4.1-12.el8.noarch 2/6 2023-11-17T05:18:11.798 INFO:teuthology.orchestra.run.smithi066.stdout: Installing : python3-jwt-1.6.1-2.el8.noarch 3/6 2023-11-17T05:18:11.798 INFO:teuthology.orchestra.run.smithi066.stdout: Installing : ceph-prometheus-alerts-2:16.2.4-0.el8.noarch 4/6 2023-11-17T05:18:11.923 INFO:teuthology.orchestra.run.smithi066.stdout: Installing : ceph-grafana-dashboards-2:16.2.4-0.el8.noarch 5/6 2023-11-17T05:18:12.180 INFO:teuthology.orchestra.run.smithi066.stdout: Installing : ceph-mgr-dashboard-2:16.2.4-0.el8.noarch 6/6 2023-11-17T05:18:12.380 INFO:teuthology.orchestra.run.smithi040.stdout: Verifying : ceph-immutable-object-cache-2:16.2.4-0.el8.x86_64 1/1 2023-11-17T05:18:12.380 INFO:teuthology.orchestra.run.smithi040.stdout: 2023-11-17T05:18:12.380 INFO:teuthology.orchestra.run.smithi040.stdout:Installed: 2023-11-17T05:18:12.380 INFO:teuthology.orchestra.run.smithi040.stdout: ceph-immutable-object-cache-2:16.2.4-0.el8.x86_64 2023-11-17T05:18:12.380 INFO:teuthology.orchestra.run.smithi040.stdout: 2023-11-17T05:18:12.380 INFO:teuthology.orchestra.run.smithi040.stdout:Complete! 2023-11-17T05:18:12.513 INFO:teuthology.orchestra.run.smithi066.stdout: Running scriptlet: ceph-mgr-dashboard-2:16.2.4-0.el8.noarch 6/6 2023-11-17T05:18:12.513 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : ceph-grafana-dashboards-2:16.2.4-0.el8.noarch 1/6 2023-11-17T05:18:12.513 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : ceph-mgr-dashboard-2:16.2.4-0.el8.noarch 2/6 2023-11-17T05:18:12.513 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : ceph-prometheus-alerts-2:16.2.4-0.el8.noarch 3/6 2023-11-17T05:18:12.513 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : python3-jwt-1.6.1-2.el8.noarch 4/6 2023-11-17T05:18:12.513 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : python3-repoze-lru-0.7-6.el8.noarch 5/6 2023-11-17T05:18:12.617 DEBUG:teuthology.orchestra.run.smithi040:> sudo yum -y install ceph-mgr 2023-11-17T05:18:13.239 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : python3-routes-2.4.1-12.el8.noarch 6/6 2023-11-17T05:18:13.239 INFO:teuthology.orchestra.run.smithi066.stdout: 2023-11-17T05:18:13.239 INFO:teuthology.orchestra.run.smithi066.stdout:Installed: 2023-11-17T05:18:13.239 INFO:teuthology.orchestra.run.smithi066.stdout: ceph-grafana-dashboards-2:16.2.4-0.el8.noarch 2023-11-17T05:18:13.239 INFO:teuthology.orchestra.run.smithi066.stdout: ceph-mgr-dashboard-2:16.2.4-0.el8.noarch 2023-11-17T05:18:13.239 INFO:teuthology.orchestra.run.smithi066.stdout: ceph-prometheus-alerts-2:16.2.4-0.el8.noarch 2023-11-17T05:18:13.239 INFO:teuthology.orchestra.run.smithi066.stdout: python3-jwt-1.6.1-2.el8.noarch 2023-11-17T05:18:13.240 INFO:teuthology.orchestra.run.smithi066.stdout: python3-repoze-lru-0.7-6.el8.noarch 2023-11-17T05:18:13.240 INFO:teuthology.orchestra.run.smithi066.stdout: python3-routes-2.4.1-12.el8.noarch 2023-11-17T05:18:13.240 INFO:teuthology.orchestra.run.smithi066.stdout: 2023-11-17T05:18:13.240 INFO:teuthology.orchestra.run.smithi066.stdout:Complete! 2023-11-17T05:18:13.372 DEBUG:teuthology.orchestra.run.smithi066:> sudo yum -y install ceph-mgr-diskprediction-local 2023-11-17T05:18:13.613 INFO:teuthology.orchestra.run.smithi040.stdout:Last metadata expiration check: 0:02:55 ago on Fri 17 Nov 2023 05:15:18 AM UTC. 2023-11-17T05:18:14.267 INFO:teuthology.orchestra.run.smithi066.stdout:Last metadata expiration check: 0:03:01 ago on Fri 17 Nov 2023 05:15:13 AM UTC. 2023-11-17T05:18:15.221 INFO:teuthology.orchestra.run.smithi040.stdout:Package ceph-mgr-2:16.2.4-0.el8.x86_64 is already installed. 2023-11-17T05:18:15.467 INFO:teuthology.orchestra.run.smithi040.stdout:Dependencies resolved. 2023-11-17T05:18:15.468 INFO:teuthology.orchestra.run.smithi040.stdout:Nothing to do. 2023-11-17T05:18:15.468 INFO:teuthology.orchestra.run.smithi040.stdout:Complete! 2023-11-17T05:18:15.607 DEBUG:teuthology.orchestra.run.smithi040:> sudo yum -y install ceph-mgr-dashboard 2023-11-17T05:18:16.051 INFO:teuthology.orchestra.run.smithi066.stdout:Dependencies resolved. 2023-11-17T05:18:16.051 INFO:teuthology.orchestra.run.smithi066.stdout:================================================================================================ 2023-11-17T05:18:16.051 INFO:teuthology.orchestra.run.smithi066.stdout: Package Arch Version Repository Size 2023-11-17T05:18:16.051 INFO:teuthology.orchestra.run.smithi066.stdout:================================================================================================ 2023-11-17T05:18:16.051 INFO:teuthology.orchestra.run.smithi066.stdout:Installing: 2023-11-17T05:18:16.051 INFO:teuthology.orchestra.run.smithi066.stdout: ceph-mgr-diskprediction-local noarch 2:16.2.4-0.el8 ceph-noarch 8.6 M 2023-11-17T05:18:16.051 INFO:teuthology.orchestra.run.smithi066.stdout:Installing dependencies: 2023-11-17T05:18:16.051 INFO:teuthology.orchestra.run.smithi066.stdout: libgfortran x86_64 8.5.0-21.el8 baseos 645 k 2023-11-17T05:18:16.051 INFO:teuthology.orchestra.run.smithi066.stdout: libquadmath x86_64 8.5.0-21.el8 baseos 172 k 2023-11-17T05:18:16.051 INFO:teuthology.orchestra.run.smithi066.stdout: openblas x86_64 0.3.15-4.el8 appstream 4.7 M 2023-11-17T05:18:16.052 INFO:teuthology.orchestra.run.smithi066.stdout: openblas-threads x86_64 0.3.15-4.el8 appstream 4.9 M 2023-11-17T05:18:16.052 INFO:teuthology.orchestra.run.smithi066.stdout: python3-numpy x86_64 1:1.14.3-10.el8 appstream 3.7 M 2023-11-17T05:18:16.052 INFO:teuthology.orchestra.run.smithi066.stdout: python3-numpy-f2py x86_64 1:1.14.3-10.el8 appstream 225 k 2023-11-17T05:18:16.052 INFO:teuthology.orchestra.run.smithi066.stdout: python3-scipy x86_64 1.0.0-21.module_el8.5.0+771+e5d9a225 appstream 14 M 2023-11-17T05:18:16.052 INFO:teuthology.orchestra.run.smithi066.stdout: 2023-11-17T05:18:16.052 INFO:teuthology.orchestra.run.smithi066.stdout:Transaction Summary 2023-11-17T05:18:16.052 INFO:teuthology.orchestra.run.smithi066.stdout:================================================================================================ 2023-11-17T05:18:16.052 INFO:teuthology.orchestra.run.smithi066.stdout:Install 8 Packages 2023-11-17T05:18:16.052 INFO:teuthology.orchestra.run.smithi066.stdout: 2023-11-17T05:18:16.052 INFO:teuthology.orchestra.run.smithi066.stdout:Total download size: 37 M 2023-11-17T05:18:16.052 INFO:teuthology.orchestra.run.smithi066.stdout:Installed size: 210 M 2023-11-17T05:18:16.052 INFO:teuthology.orchestra.run.smithi066.stdout:Downloading Packages: 2023-11-17T05:18:16.426 INFO:teuthology.orchestra.run.smithi066.stdout:(1/8): ceph-mgr-diskprediction-local-16.2.4-0.e 57 MB/s | 8.6 MB 00:00 2023-11-17T05:18:16.582 INFO:teuthology.orchestra.run.smithi040.stdout:Last metadata expiration check: 0:02:58 ago on Fri 17 Nov 2023 05:15:18 AM UTC. 2023-11-17T05:18:16.751 INFO:teuthology.orchestra.run.smithi066.stdout:(2/8): openblas-0.3.15-4.el8.x86_64.rpm 9.9 MB/s | 4.7 MB 00:00 2023-11-17T05:18:16.843 INFO:teuthology.orchestra.run.smithi066.stdout:(3/8): openblas-threads-0.3.15-4.el8.x86_64.rpm 8.5 MB/s | 4.9 MB 00:00 2023-11-17T05:18:16.870 INFO:teuthology.orchestra.run.smithi066.stdout:(4/8): python3-numpy-f2py-1.14.3-10.el8.x86_64. 1.9 MB/s | 225 kB 00:00 2023-11-17T05:18:16.935 INFO:teuthology.orchestra.run.smithi066.stdout:(5/8): python3-numpy-1.14.3-10.el8.x86_64.rpm 7.3 MB/s | 3.7 MB 00:00 2023-11-17T05:18:17.052 INFO:teuthology.orchestra.run.smithi066.stdout:(6/8): libquadmath-8.5.0-21.el8.x86_64.rpm 1.4 MB/s | 172 kB 00:00 2023-11-17T05:18:17.085 INFO:teuthology.orchestra.run.smithi066.stdout:(7/8): libgfortran-8.5.0-21.el8.x86_64.rpm 2.9 MB/s | 645 kB 00:00 2023-11-17T05:18:17.428 INFO:teuthology.orchestra.run.smithi066.stdout:(8/8): python3-scipy-1.0.0-21.module_el8.5.0+77 23 MB/s | 14 MB 00:00 2023-11-17T05:18:17.428 INFO:teuthology.orchestra.run.smithi066.stdout:-------------------------------------------------------------------------------- 2023-11-17T05:18:17.428 INFO:teuthology.orchestra.run.smithi066.stdout:Total 27 MB/s | 37 MB 00:01 2023-11-17T05:18:17.666 INFO:teuthology.orchestra.run.smithi066.stdout:Running transaction check 2023-11-17T05:18:17.685 INFO:teuthology.orchestra.run.smithi066.stdout:Transaction check succeeded. 2023-11-17T05:18:17.685 INFO:teuthology.orchestra.run.smithi066.stdout:Running transaction test 2023-11-17T05:18:18.042 INFO:teuthology.orchestra.run.smithi066.stdout:Transaction test succeeded. 2023-11-17T05:18:18.045 INFO:teuthology.orchestra.run.smithi066.stdout:Running transaction 2023-11-17T05:18:18.438 INFO:teuthology.orchestra.run.smithi040.stdout:Dependencies resolved. 2023-11-17T05:18:18.439 INFO:teuthology.orchestra.run.smithi040.stdout:================================================================================ 2023-11-17T05:18:18.439 INFO:teuthology.orchestra.run.smithi040.stdout: Package Arch Version Repository Size 2023-11-17T05:18:18.440 INFO:teuthology.orchestra.run.smithi040.stdout:================================================================================ 2023-11-17T05:18:18.440 INFO:teuthology.orchestra.run.smithi040.stdout:Installing: 2023-11-17T05:18:18.440 INFO:teuthology.orchestra.run.smithi040.stdout: ceph-mgr-dashboard noarch 2:16.2.4-0.el8 ceph-noarch 2.7 M 2023-11-17T05:18:18.440 INFO:teuthology.orchestra.run.smithi040.stdout:Installing dependencies: 2023-11-17T05:18:18.440 INFO:teuthology.orchestra.run.smithi040.stdout: ceph-grafana-dashboards noarch 2:16.2.4-0.el8 ceph-noarch 24 k 2023-11-17T05:18:18.440 INFO:teuthology.orchestra.run.smithi040.stdout: ceph-prometheus-alerts noarch 2:16.2.4-0.el8 ceph-noarch 8.9 k 2023-11-17T05:18:18.440 INFO:teuthology.orchestra.run.smithi040.stdout: python3-jwt noarch 1.6.1-2.el8 baseos 43 k 2023-11-17T05:18:18.440 INFO:teuthology.orchestra.run.smithi040.stdout: python3-repoze-lru noarch 0.7-6.el8 epel 34 k 2023-11-17T05:18:18.440 INFO:teuthology.orchestra.run.smithi040.stdout: python3-routes noarch 2.4.1-12.el8 epel 196 k 2023-11-17T05:18:18.440 INFO:teuthology.orchestra.run.smithi040.stdout: 2023-11-17T05:18:18.440 INFO:teuthology.orchestra.run.smithi040.stdout:Transaction Summary 2023-11-17T05:18:18.440 INFO:teuthology.orchestra.run.smithi040.stdout:================================================================================ 2023-11-17T05:18:18.441 INFO:teuthology.orchestra.run.smithi040.stdout:Install 6 Packages 2023-11-17T05:18:18.441 INFO:teuthology.orchestra.run.smithi040.stdout: 2023-11-17T05:18:18.441 INFO:teuthology.orchestra.run.smithi040.stdout:Total download size: 3.0 M 2023-11-17T05:18:18.441 INFO:teuthology.orchestra.run.smithi040.stdout:Installed size: 15 M 2023-11-17T05:18:18.441 INFO:teuthology.orchestra.run.smithi040.stdout:Downloading Packages: 2023-11-17T05:18:18.706 INFO:teuthology.orchestra.run.smithi066.stdout: Preparing : 1/1 2023-11-17T05:18:18.712 INFO:teuthology.orchestra.run.smithi040.stdout:(1/6): ceph-grafana-dashboards-16.2.4-0.el8.noa 462 kB/s | 24 kB 00:00 2023-11-17T05:18:18.738 INFO:teuthology.orchestra.run.smithi040.stdout:(2/6): ceph-prometheus-alerts-16.2.4-0.el8.noar 116 kB/s | 8.9 kB 00:00 2023-11-17T05:18:18.813 INFO:teuthology.orchestra.run.smithi040.stdout:(3/6): ceph-mgr-dashboard-16.2.4-0.el8.noarch.r 17 MB/s | 2.7 MB 00:00 2023-11-17T05:18:18.850 INFO:teuthology.orchestra.run.smithi066.stdout: Installing : libquadmath-8.5.0-21.el8.x86_64 1/8 2023-11-17T05:18:18.897 INFO:teuthology.orchestra.run.smithi040.stdout:(4/6): python3-jwt-1.6.1-2.el8.noarch.rpm 232 kB/s | 43 kB 00:00 2023-11-17T05:18:18.922 INFO:teuthology.orchestra.run.smithi040.stdout:(5/6): python3-repoze-lru-0.7-6.el8.noarch.rpm 183 kB/s | 34 kB 00:00 2023-11-17T05:18:18.989 INFO:teuthology.orchestra.run.smithi066.stdout: Running scriptlet: libquadmath-8.5.0-21.el8.x86_64 1/8 2023-11-17T05:18:19.022 INFO:teuthology.orchestra.run.smithi040.stdout:(6/6): python3-routes-2.4.1-12.el8.noarch.rpm 966 kB/s | 196 kB 00:00 2023-11-17T05:18:19.026 INFO:teuthology.orchestra.run.smithi040.stdout:-------------------------------------------------------------------------------- 2023-11-17T05:18:19.026 INFO:teuthology.orchestra.run.smithi040.stdout:Total 5.0 MB/s | 3.0 MB 00:00 2023-11-17T05:18:19.039 INFO:teuthology.orchestra.run.smithi040.stdout:Running transaction check 2023-11-17T05:18:19.051 INFO:teuthology.orchestra.run.smithi040.stdout:Transaction check succeeded. 2023-11-17T05:18:19.051 INFO:teuthology.orchestra.run.smithi040.stdout:Running transaction test 2023-11-17T05:18:19.125 INFO:teuthology.orchestra.run.smithi066.stdout: Installing : libgfortran-8.5.0-21.el8.x86_64 2/8 2023-11-17T05:18:19.212 INFO:teuthology.orchestra.run.smithi040.stdout:Transaction test succeeded. 2023-11-17T05:18:19.215 INFO:teuthology.orchestra.run.smithi040.stdout:Running transaction 2023-11-17T05:18:19.520 INFO:teuthology.orchestra.run.smithi040.stdout: Preparing : 1/1 2023-11-17T05:18:19.744 INFO:teuthology.orchestra.run.smithi066.stdout: Running scriptlet: libgfortran-8.5.0-21.el8.x86_64 2/8 2023-11-17T05:18:19.760 INFO:teuthology.orchestra.run.smithi040.stdout: Installing : python3-repoze-lru-0.7-6.el8.noarch 1/6 2023-11-17T05:18:19.933 INFO:teuthology.orchestra.run.smithi066.stdout: Installing : openblas-0.3.15-4.el8.x86_64 3/8 2023-11-17T05:18:20.024 INFO:teuthology.orchestra.run.smithi040.stdout: Installing : python3-routes-2.4.1-12.el8.noarch 2/6 2023-11-17T05:18:20.213 INFO:teuthology.orchestra.run.smithi040.stdout: Installing : python3-jwt-1.6.1-2.el8.noarch 3/6 2023-11-17T05:18:20.369 INFO:teuthology.orchestra.run.smithi040.stdout: Installing : ceph-prometheus-alerts-2:16.2.4-0.el8.noarch 4/6 2023-11-17T05:18:20.581 INFO:teuthology.orchestra.run.smithi066.stdout: Running scriptlet: openblas-0.3.15-4.el8.x86_64 3/8 2023-11-17T05:18:20.761 INFO:teuthology.orchestra.run.smithi066.stdout: Installing : openblas-threads-0.3.15-4.el8.x86_64 4/8 2023-11-17T05:18:21.490 INFO:teuthology.orchestra.run.smithi040.stdout: Installing : ceph-grafana-dashboards-2:16.2.4-0.el8.noarch 5/6 2023-11-17T05:18:21.497 INFO:teuthology.orchestra.run.smithi066.stdout: Running scriptlet: openblas-threads-0.3.15-4.el8.x86_64 4/8 2023-11-17T05:18:21.804 INFO:teuthology.orchestra.run.smithi040.stdout: Installing : ceph-mgr-dashboard-2:16.2.4-0.el8.noarch 6/6 2023-11-17T05:18:21.866 INFO:teuthology.orchestra.run.smithi066.stdout: Installing : python3-numpy-1:1.14.3-10.el8.x86_64 5/8 2023-11-17T05:18:22.099 INFO:teuthology.orchestra.run.smithi066.stdout: Installing : python3-numpy-f2py-1:1.14.3-10.el8.x86_64 6/8 2023-11-17T05:18:22.128 INFO:teuthology.orchestra.run.smithi040.stdout: Running scriptlet: ceph-mgr-dashboard-2:16.2.4-0.el8.noarch 6/6 2023-11-17T05:18:22.128 INFO:teuthology.orchestra.run.smithi040.stdout: Verifying : ceph-grafana-dashboards-2:16.2.4-0.el8.noarch 1/6 2023-11-17T05:18:22.129 INFO:teuthology.orchestra.run.smithi040.stdout: Verifying : ceph-mgr-dashboard-2:16.2.4-0.el8.noarch 2/6 2023-11-17T05:18:22.129 INFO:teuthology.orchestra.run.smithi040.stdout: Verifying : ceph-prometheus-alerts-2:16.2.4-0.el8.noarch 3/6 2023-11-17T05:18:22.129 INFO:teuthology.orchestra.run.smithi040.stdout: Verifying : python3-jwt-1.6.1-2.el8.noarch 4/6 2023-11-17T05:18:22.129 INFO:teuthology.orchestra.run.smithi040.stdout: Verifying : python3-repoze-lru-0.7-6.el8.noarch 5/6 2023-11-17T05:18:22.873 INFO:teuthology.orchestra.run.smithi040.stdout: Verifying : python3-routes-2.4.1-12.el8.noarch 6/6 2023-11-17T05:18:22.873 INFO:teuthology.orchestra.run.smithi040.stdout: 2023-11-17T05:18:22.873 INFO:teuthology.orchestra.run.smithi040.stdout:Installed: 2023-11-17T05:18:22.873 INFO:teuthology.orchestra.run.smithi040.stdout: ceph-grafana-dashboards-2:16.2.4-0.el8.noarch 2023-11-17T05:18:22.873 INFO:teuthology.orchestra.run.smithi040.stdout: ceph-mgr-dashboard-2:16.2.4-0.el8.noarch 2023-11-17T05:18:22.873 INFO:teuthology.orchestra.run.smithi040.stdout: ceph-prometheus-alerts-2:16.2.4-0.el8.noarch 2023-11-17T05:18:22.873 INFO:teuthology.orchestra.run.smithi040.stdout: python3-jwt-1.6.1-2.el8.noarch 2023-11-17T05:18:22.873 INFO:teuthology.orchestra.run.smithi040.stdout: python3-repoze-lru-0.7-6.el8.noarch 2023-11-17T05:18:22.873 INFO:teuthology.orchestra.run.smithi040.stdout: python3-routes-2.4.1-12.el8.noarch 2023-11-17T05:18:22.873 INFO:teuthology.orchestra.run.smithi040.stdout: 2023-11-17T05:18:22.874 INFO:teuthology.orchestra.run.smithi040.stdout:Complete! 2023-11-17T05:18:23.022 DEBUG:teuthology.orchestra.run.smithi040:> sudo yum -y install ceph-mgr-diskprediction-local 2023-11-17T05:18:23.971 INFO:teuthology.orchestra.run.smithi066.stdout: Running scriptlet: python3-numpy-f2py-1:1.14.3-10.el8.x86_64 6/8 2023-11-17T05:18:24.019 INFO:teuthology.orchestra.run.smithi040.stdout:Last metadata expiration check: 0:03:06 ago on Fri 17 Nov 2023 05:15:18 AM UTC. 2023-11-17T05:18:25.369 INFO:teuthology.orchestra.run.smithi066.stdout: Installing : python3-scipy-1.0.0-21.module_el8.5.0+771+e5d9a225.x 7/8 2023-11-17T05:18:25.549 INFO:teuthology.orchestra.run.smithi066.stdout: Installing : ceph-mgr-diskprediction-local-2:16.2.4-0.el8.noarch 8/8 2023-11-17T05:18:25.878 INFO:teuthology.orchestra.run.smithi040.stdout:Dependencies resolved. 2023-11-17T05:18:25.880 INFO:teuthology.orchestra.run.smithi040.stdout:================================================================================================ 2023-11-17T05:18:25.880 INFO:teuthology.orchestra.run.smithi040.stdout: Package Arch Version Repository Size 2023-11-17T05:18:25.880 INFO:teuthology.orchestra.run.smithi040.stdout:================================================================================================ 2023-11-17T05:18:25.880 INFO:teuthology.orchestra.run.smithi040.stdout:Installing: 2023-11-17T05:18:25.881 INFO:teuthology.orchestra.run.smithi040.stdout: ceph-mgr-diskprediction-local noarch 2:16.2.4-0.el8 ceph-noarch 8.6 M 2023-11-17T05:18:25.881 INFO:teuthology.orchestra.run.smithi040.stdout:Installing dependencies: 2023-11-17T05:18:25.881 INFO:teuthology.orchestra.run.smithi040.stdout: libgfortran x86_64 8.5.0-21.el8 baseos 645 k 2023-11-17T05:18:25.881 INFO:teuthology.orchestra.run.smithi040.stdout: libquadmath x86_64 8.5.0-21.el8 baseos 172 k 2023-11-17T05:18:25.882 INFO:teuthology.orchestra.run.smithi040.stdout: openblas x86_64 0.3.15-4.el8 appstream 4.7 M 2023-11-17T05:18:25.882 INFO:teuthology.orchestra.run.smithi040.stdout: openblas-threads x86_64 0.3.15-4.el8 appstream 4.9 M 2023-11-17T05:18:25.882 INFO:teuthology.orchestra.run.smithi040.stdout: python3-numpy x86_64 1:1.14.3-10.el8 appstream 3.7 M 2023-11-17T05:18:25.882 INFO:teuthology.orchestra.run.smithi040.stdout: python3-numpy-f2py x86_64 1:1.14.3-10.el8 appstream 225 k 2023-11-17T05:18:25.882 INFO:teuthology.orchestra.run.smithi040.stdout: python3-scipy x86_64 1.0.0-21.module_el8.5.0+771+e5d9a225 appstream 14 M 2023-11-17T05:18:25.883 INFO:teuthology.orchestra.run.smithi040.stdout: 2023-11-17T05:18:25.883 INFO:teuthology.orchestra.run.smithi040.stdout:Transaction Summary 2023-11-17T05:18:25.883 INFO:teuthology.orchestra.run.smithi040.stdout:================================================================================================ 2023-11-17T05:18:25.883 INFO:teuthology.orchestra.run.smithi040.stdout:Install 8 Packages 2023-11-17T05:18:25.883 INFO:teuthology.orchestra.run.smithi040.stdout: 2023-11-17T05:18:25.884 INFO:teuthology.orchestra.run.smithi040.stdout:Total download size: 37 M 2023-11-17T05:18:25.884 INFO:teuthology.orchestra.run.smithi040.stdout:Installed size: 210 M 2023-11-17T05:18:25.884 INFO:teuthology.orchestra.run.smithi040.stdout:Downloading Packages: 2023-11-17T05:18:25.890 INFO:teuthology.orchestra.run.smithi066.stdout: Running scriptlet: ceph-mgr-diskprediction-local-2:16.2.4-0.el8.noarch 8/8 2023-11-17T05:18:25.890 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : ceph-mgr-diskprediction-local-2:16.2.4-0.el8.noarch 1/8 2023-11-17T05:18:25.890 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : openblas-0.3.15-4.el8.x86_64 2/8 2023-11-17T05:18:25.891 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : openblas-threads-0.3.15-4.el8.x86_64 3/8 2023-11-17T05:18:25.891 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : python3-numpy-1:1.14.3-10.el8.x86_64 4/8 2023-11-17T05:18:25.891 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : python3-numpy-f2py-1:1.14.3-10.el8.x86_64 5/8 2023-11-17T05:18:25.891 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : python3-scipy-1.0.0-21.module_el8.5.0+771+e5d9a225.x 6/8 2023-11-17T05:18:25.891 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : libgfortran-8.5.0-21.el8.x86_64 7/8 2023-11-17T05:18:26.429 INFO:teuthology.orchestra.run.smithi040.stdout:(1/8): ceph-mgr-diskprediction-local-16.2.4-0.e 48 MB/s | 8.6 MB 00:00 2023-11-17T05:18:26.639 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : libquadmath-8.5.0-21.el8.x86_64 8/8 2023-11-17T05:18:26.639 INFO:teuthology.orchestra.run.smithi066.stdout: 2023-11-17T05:18:26.639 INFO:teuthology.orchestra.run.smithi066.stdout:Installed: 2023-11-17T05:18:26.639 INFO:teuthology.orchestra.run.smithi066.stdout: ceph-mgr-diskprediction-local-2:16.2.4-0.el8.noarch 2023-11-17T05:18:26.639 INFO:teuthology.orchestra.run.smithi066.stdout: libgfortran-8.5.0-21.el8.x86_64 2023-11-17T05:18:26.640 INFO:teuthology.orchestra.run.smithi066.stdout: libquadmath-8.5.0-21.el8.x86_64 2023-11-17T05:18:26.640 INFO:teuthology.orchestra.run.smithi066.stdout: openblas-0.3.15-4.el8.x86_64 2023-11-17T05:18:26.640 INFO:teuthology.orchestra.run.smithi066.stdout: openblas-threads-0.3.15-4.el8.x86_64 2023-11-17T05:18:26.640 INFO:teuthology.orchestra.run.smithi066.stdout: python3-numpy-1:1.14.3-10.el8.x86_64 2023-11-17T05:18:26.640 INFO:teuthology.orchestra.run.smithi066.stdout: python3-numpy-f2py-1:1.14.3-10.el8.x86_64 2023-11-17T05:18:26.640 INFO:teuthology.orchestra.run.smithi066.stdout: python3-scipy-1.0.0-21.module_el8.5.0+771+e5d9a225.x86_64 2023-11-17T05:18:26.640 INFO:teuthology.orchestra.run.smithi066.stdout: 2023-11-17T05:18:26.640 INFO:teuthology.orchestra.run.smithi066.stdout:Complete! 2023-11-17T05:18:26.731 INFO:teuthology.orchestra.run.smithi040.stdout:(2/8): openblas-threads-0.3.15-4.el8.x86_64.rpm 10 MB/s | 4.9 MB 00:00 2023-11-17T05:18:26.753 DEBUG:teuthology.orchestra.run.smithi066:> sudo yum -y install ceph-mgr-rook 2023-11-17T05:18:26.796 INFO:teuthology.orchestra.run.smithi040.stdout:(3/8): openblas-0.3.15-4.el8.x86_64.rpm 8.6 MB/s | 4.7 MB 00:00 2023-11-17T05:18:26.821 INFO:teuthology.orchestra.run.smithi040.stdout:(4/8): python3-numpy-f2py-1.14.3-10.el8.x86_64. 2.4 MB/s | 225 kB 00:00 2023-11-17T05:18:26.988 INFO:teuthology.orchestra.run.smithi040.stdout:(5/8): libgfortran-8.5.0-21.el8.x86_64.rpm 3.8 MB/s | 645 kB 00:00 2023-11-17T05:18:27.030 INFO:teuthology.orchestra.run.smithi040.stdout:(6/8): libquadmath-8.5.0-21.el8.x86_64.rpm 4.1 MB/s | 172 kB 00:00 2023-11-17T05:18:27.131 INFO:teuthology.orchestra.run.smithi040.stdout:(7/8): python3-numpy-1.14.3-10.el8.x86_64.rpm 5.3 MB/s | 3.7 MB 00:00 2023-11-17T05:18:27.381 INFO:teuthology.orchestra.run.smithi040.stdout:(8/8): python3-scipy-1.0.0-21.module_el8.5.0+77 23 MB/s | 14 MB 00:00 2023-11-17T05:18:27.381 INFO:teuthology.orchestra.run.smithi040.stdout:-------------------------------------------------------------------------------- 2023-11-17T05:18:27.382 INFO:teuthology.orchestra.run.smithi040.stdout:Total 24 MB/s | 37 MB 00:01 2023-11-17T05:18:27.633 INFO:teuthology.orchestra.run.smithi040.stdout:Running transaction check 2023-11-17T05:18:27.649 INFO:teuthology.orchestra.run.smithi066.stdout:Last metadata expiration check: 0:03:14 ago on Fri 17 Nov 2023 05:15:13 AM UTC. 2023-11-17T05:18:27.653 INFO:teuthology.orchestra.run.smithi040.stdout:Transaction check succeeded. 2023-11-17T05:18:27.653 INFO:teuthology.orchestra.run.smithi040.stdout:Running transaction test 2023-11-17T05:18:28.007 INFO:teuthology.orchestra.run.smithi040.stdout:Transaction test succeeded. 2023-11-17T05:18:28.010 INFO:teuthology.orchestra.run.smithi040.stdout:Running transaction 2023-11-17T05:18:28.702 INFO:teuthology.orchestra.run.smithi040.stdout: Preparing : 1/1 2023-11-17T05:18:28.842 INFO:teuthology.orchestra.run.smithi040.stdout: Installing : libquadmath-8.5.0-21.el8.x86_64 1/8 2023-11-17T05:18:28.976 INFO:teuthology.orchestra.run.smithi040.stdout: Running scriptlet: libquadmath-8.5.0-21.el8.x86_64 1/8 2023-11-17T05:18:29.100 INFO:teuthology.orchestra.run.smithi040.stdout: Installing : libgfortran-8.5.0-21.el8.x86_64 2/8 2023-11-17T05:18:29.438 INFO:teuthology.orchestra.run.smithi066.stdout:Dependencies resolved. 2023-11-17T05:18:29.440 INFO:teuthology.orchestra.run.smithi066.stdout:================================================================================ 2023-11-17T05:18:29.441 INFO:teuthology.orchestra.run.smithi066.stdout: Package Arch Version Repository Size 2023-11-17T05:18:29.441 INFO:teuthology.orchestra.run.smithi066.stdout:================================================================================ 2023-11-17T05:18:29.441 INFO:teuthology.orchestra.run.smithi066.stdout:Installing: 2023-11-17T05:18:29.441 INFO:teuthology.orchestra.run.smithi066.stdout: ceph-mgr-rook noarch 2:16.2.4-0.el8 ceph-noarch 27 k 2023-11-17T05:18:29.441 INFO:teuthology.orchestra.run.smithi066.stdout:Installing dependencies: 2023-11-17T05:18:29.441 INFO:teuthology.orchestra.run.smithi066.stdout: python3-cachetools noarch 3.1.1-4.el8 epel 33 k 2023-11-17T05:18:29.441 INFO:teuthology.orchestra.run.smithi066.stdout: python3-certifi noarch 2018.10.15-7.el8 epel 16 k 2023-11-17T05:18:29.441 INFO:teuthology.orchestra.run.smithi066.stdout: python3-google-auth noarch 1:1.1.1-10.el8 epel 81 k 2023-11-17T05:18:29.441 INFO:teuthology.orchestra.run.smithi066.stdout: python3-jsonpatch noarch 1.21-2.el8 appstream 27 k 2023-11-17T05:18:29.441 INFO:teuthology.orchestra.run.smithi066.stdout: python3-jsonpointer noarch 1.10-11.el8 appstream 20 k 2023-11-17T05:18:29.441 INFO:teuthology.orchestra.run.smithi066.stdout: python3-kubernetes noarch 1:11.0.0-6.el8 epel 1.2 M 2023-11-17T05:18:29.441 INFO:teuthology.orchestra.run.smithi066.stdout: python3-oauthlib noarch 2.1.0-1.el8 baseos 155 k 2023-11-17T05:18:29.441 INFO:teuthology.orchestra.run.smithi066.stdout: python3-pyasn1 noarch 0.3.7-6.el8 appstream 126 k 2023-11-17T05:18:29.442 INFO:teuthology.orchestra.run.smithi066.stdout: python3-pyasn1-modules noarch 0.3.7-6.el8 appstream 110 k 2023-11-17T05:18:29.442 INFO:teuthology.orchestra.run.smithi066.stdout: python3-requests-oauthlib noarch 1.0.0-1.el8 baseos 43 k 2023-11-17T05:18:29.442 INFO:teuthology.orchestra.run.smithi066.stdout: python3-rsa noarch 4.9-2.el8 epel 64 k 2023-11-17T05:18:29.442 INFO:teuthology.orchestra.run.smithi066.stdout: python3-websocket-client noarch 0.56.0-5.el8 epel 61 k 2023-11-17T05:18:29.442 INFO:teuthology.orchestra.run.smithi066.stdout: 2023-11-17T05:18:29.442 INFO:teuthology.orchestra.run.smithi066.stdout:Transaction Summary 2023-11-17T05:18:29.442 INFO:teuthology.orchestra.run.smithi066.stdout:================================================================================ 2023-11-17T05:18:29.442 INFO:teuthology.orchestra.run.smithi066.stdout:Install 13 Packages 2023-11-17T05:18:29.442 INFO:teuthology.orchestra.run.smithi066.stdout: 2023-11-17T05:18:29.442 INFO:teuthology.orchestra.run.smithi066.stdout:Total download size: 1.9 M 2023-11-17T05:18:29.442 INFO:teuthology.orchestra.run.smithi066.stdout:Installed size: 24 M 2023-11-17T05:18:29.443 INFO:teuthology.orchestra.run.smithi066.stdout:Downloading Packages: 2023-11-17T05:18:29.763 INFO:teuthology.orchestra.run.smithi040.stdout: Running scriptlet: libgfortran-8.5.0-21.el8.x86_64 2/8 2023-11-17T05:18:29.942 INFO:teuthology.orchestra.run.smithi040.stdout: Installing : openblas-0.3.15-4.el8.x86_64 3/8 2023-11-17T05:18:30.079 INFO:teuthology.orchestra.run.smithi066.stdout:(1/13): ceph-mgr-rook-16.2.4-0.el8.noarch.rpm 370 kB/s | 27 kB 00:00 2023-11-17T05:18:30.167 INFO:teuthology.orchestra.run.smithi066.stdout:(2/13): python3-jsonpointer-1.10-11.el8.noarch. 124 kB/s | 20 kB 00:00 2023-11-17T05:18:30.193 INFO:teuthology.orchestra.run.smithi066.stdout:(3/13): python3-jsonpatch-1.21-2.el8.noarch.rpm 143 kB/s | 27 kB 00:00 2023-11-17T05:18:30.259 INFO:teuthology.orchestra.run.smithi066.stdout:(4/13): python3-pyasn1-0.3.7-6.el8.noarch.rpm 702 kB/s | 126 kB 00:00 2023-11-17T05:18:30.293 INFO:teuthology.orchestra.run.smithi066.stdout:(5/13): python3-pyasn1-modules-0.3.7-6.el8.noar 877 kB/s | 110 kB 00:00 2023-11-17T05:18:30.318 INFO:teuthology.orchestra.run.smithi066.stdout:(6/13): python3-oauthlib-2.1.0-1.el8.noarch.rpm 1.2 MB/s | 155 kB 00:00 2023-11-17T05:18:30.369 INFO:teuthology.orchestra.run.smithi066.stdout:(7/13): python3-requests-oauthlib-1.0.0-1.el8.n 391 kB/s | 43 kB 00:00 2023-11-17T05:18:30.598 INFO:teuthology.orchestra.run.smithi040.stdout: Running scriptlet: openblas-0.3.15-4.el8.x86_64 3/8 2023-11-17T05:18:30.619 INFO:teuthology.orchestra.run.smithi066.stdout:(8/13): python3-certifi-2018.10.15-7.el8.noarch 54 kB/s | 16 kB 00:00 2023-11-17T05:18:30.711 INFO:teuthology.orchestra.run.smithi066.stdout:(9/13): python3-cachetools-3.1.1-4.el8.noarch.r 79 kB/s | 33 kB 00:00 2023-11-17T05:18:30.775 INFO:teuthology.orchestra.run.smithi040.stdout: Installing : openblas-threads-0.3.15-4.el8.x86_64 4/8 2023-11-17T05:18:30.849 INFO:teuthology.orchestra.run.smithi066.stdout:(10/13): python3-google-auth-1.1.1-10.el8.noarc 171 kB/s | 81 kB 00:00 2023-11-17T05:18:30.919 INFO:teuthology.orchestra.run.smithi066.stdout:(11/13): python3-rsa-4.9-2.el8.noarch.rpm 308 kB/s | 64 kB 00:00 2023-11-17T05:18:31.078 INFO:teuthology.orchestra.run.smithi066.stdout:(12/13): python3-websocket-client-0.56.0-5.el8. 262 kB/s | 61 kB 00:00 2023-11-17T05:18:31.541 INFO:teuthology.orchestra.run.smithi040.stdout: Running scriptlet: openblas-threads-0.3.15-4.el8.x86_64 4/8 2023-11-17T05:18:31.874 INFO:teuthology.orchestra.run.smithi040.stdout: Installing : python3-numpy-1:1.14.3-10.el8.x86_64 5/8 2023-11-17T05:18:32.042 INFO:teuthology.orchestra.run.smithi066.stdout:(13/13): python3-kubernetes-11.0.0-6.el8.noarch 839 kB/s | 1.2 MB 00:01 2023-11-17T05:18:32.043 INFO:teuthology.orchestra.run.smithi066.stdout:-------------------------------------------------------------------------------- 2023-11-17T05:18:32.043 INFO:teuthology.orchestra.run.smithi066.stdout:Total 752 kB/s | 1.9 MB 00:02 2023-11-17T05:18:32.103 INFO:teuthology.orchestra.run.smithi066.stdout:Running transaction check 2023-11-17T05:18:32.120 INFO:teuthology.orchestra.run.smithi066.stdout:Transaction check succeeded. 2023-11-17T05:18:32.120 INFO:teuthology.orchestra.run.smithi066.stdout:Running transaction test 2023-11-17T05:18:32.167 INFO:teuthology.orchestra.run.smithi040.stdout: Installing : python3-numpy-f2py-1:1.14.3-10.el8.x86_64 6/8 2023-11-17T05:18:32.214 INFO:teuthology.orchestra.run.smithi066.stdout:Transaction test succeeded. 2023-11-17T05:18:32.218 INFO:teuthology.orchestra.run.smithi066.stdout:Running transaction 2023-11-17T05:18:32.737 INFO:teuthology.orchestra.run.smithi066.stdout: Preparing : 1/1 2023-11-17T05:18:34.845 INFO:teuthology.orchestra.run.smithi040.stdout: Running scriptlet: python3-numpy-f2py-1:1.14.3-10.el8.x86_64 6/8 2023-11-17T05:18:34.845 INFO:teuthology.orchestra.run.smithi066.stdout: Installing : python3-pyasn1-0.3.7-6.el8.noarch 1/13 2023-11-17T05:18:34.845 INFO:teuthology.orchestra.run.smithi066.stdout: Installing : python3-pyasn1-modules-0.3.7-6.el8.noarch 2/13 2023-11-17T05:18:34.846 INFO:teuthology.orchestra.run.smithi066.stdout: Installing : python3-rsa-4.9-2.el8.noarch 3/13 2023-11-17T05:18:34.846 INFO:teuthology.orchestra.run.smithi066.stdout: Installing : python3-websocket-client-0.56.0-5.el8.noarch 4/13 2023-11-17T05:18:34.846 INFO:teuthology.orchestra.run.smithi066.stdout: Installing : python3-certifi-2018.10.15-7.el8.noarch 5/13 2023-11-17T05:18:34.846 INFO:teuthology.orchestra.run.smithi066.stdout: Installing : python3-cachetools-3.1.1-4.el8.noarch 6/13 2023-11-17T05:18:34.846 INFO:teuthology.orchestra.run.smithi066.stdout: Installing : python3-google-auth-1:1.1.1-10.el8.noarch 7/13 2023-11-17T05:18:34.846 INFO:teuthology.orchestra.run.smithi066.stdout: Installing : python3-oauthlib-2.1.0-1.el8.noarch 8/13 2023-11-17T05:18:35.592 INFO:teuthology.orchestra.run.smithi040.stdout: Installing : python3-scipy-1.0.0-21.module_el8.5.0+771+e5d9a225.x 7/8 2023-11-17T05:18:35.775 INFO:teuthology.orchestra.run.smithi040.stdout: Installing : ceph-mgr-diskprediction-local-2:16.2.4-0.el8.noarch 8/8 2023-11-17T05:18:35.944 INFO:teuthology.orchestra.run.smithi066.stdout: Installing : python3-requests-oauthlib-1.0.0-1.el8.noarch 9/13 2023-11-17T05:18:36.161 INFO:teuthology.orchestra.run.smithi040.stdout: Running scriptlet: ceph-mgr-diskprediction-local-2:16.2.4-0.el8.noarch 8/8 2023-11-17T05:18:36.161 INFO:teuthology.orchestra.run.smithi040.stdout: Verifying : ceph-mgr-diskprediction-local-2:16.2.4-0.el8.noarch 1/8 2023-11-17T05:18:36.161 INFO:teuthology.orchestra.run.smithi040.stdout: Verifying : openblas-0.3.15-4.el8.x86_64 2/8 2023-11-17T05:18:36.162 INFO:teuthology.orchestra.run.smithi040.stdout: Verifying : openblas-threads-0.3.15-4.el8.x86_64 3/8 2023-11-17T05:18:36.162 INFO:teuthology.orchestra.run.smithi040.stdout: Verifying : python3-numpy-1:1.14.3-10.el8.x86_64 4/8 2023-11-17T05:18:36.162 INFO:teuthology.orchestra.run.smithi040.stdout: Verifying : python3-numpy-f2py-1:1.14.3-10.el8.x86_64 5/8 2023-11-17T05:18:36.162 INFO:teuthology.orchestra.run.smithi040.stdout: Verifying : python3-scipy-1.0.0-21.module_el8.5.0+771+e5d9a225.x 6/8 2023-11-17T05:18:36.162 INFO:teuthology.orchestra.run.smithi040.stdout: Verifying : libgfortran-8.5.0-21.el8.x86_64 7/8 2023-11-17T05:18:36.254 INFO:teuthology.orchestra.run.smithi066.stdout: Installing : python3-kubernetes-1:11.0.0-6.el8.noarch 10/13 2023-11-17T05:18:36.523 INFO:teuthology.orchestra.run.smithi066.stdout: Installing : python3-jsonpointer-1.10-11.el8.noarch 11/13 2023-11-17T05:18:36.714 INFO:teuthology.orchestra.run.smithi066.stdout: Installing : python3-jsonpatch-1.21-2.el8.noarch 12/13 2023-11-17T05:18:36.880 INFO:teuthology.orchestra.run.smithi066.stdout: Installing : ceph-mgr-rook-2:16.2.4-0.el8.noarch 13/13 2023-11-17T05:18:36.959 INFO:teuthology.orchestra.run.smithi040.stdout: Verifying : libquadmath-8.5.0-21.el8.x86_64 8/8 2023-11-17T05:18:36.959 INFO:teuthology.orchestra.run.smithi040.stdout: 2023-11-17T05:18:36.959 INFO:teuthology.orchestra.run.smithi040.stdout:Installed: 2023-11-17T05:18:36.960 INFO:teuthology.orchestra.run.smithi040.stdout: ceph-mgr-diskprediction-local-2:16.2.4-0.el8.noarch 2023-11-17T05:18:36.960 INFO:teuthology.orchestra.run.smithi040.stdout: libgfortran-8.5.0-21.el8.x86_64 2023-11-17T05:18:36.960 INFO:teuthology.orchestra.run.smithi040.stdout: libquadmath-8.5.0-21.el8.x86_64 2023-11-17T05:18:36.960 INFO:teuthology.orchestra.run.smithi040.stdout: openblas-0.3.15-4.el8.x86_64 2023-11-17T05:18:36.960 INFO:teuthology.orchestra.run.smithi040.stdout: openblas-threads-0.3.15-4.el8.x86_64 2023-11-17T05:18:36.960 INFO:teuthology.orchestra.run.smithi040.stdout: python3-numpy-1:1.14.3-10.el8.x86_64 2023-11-17T05:18:36.960 INFO:teuthology.orchestra.run.smithi040.stdout: python3-numpy-f2py-1:1.14.3-10.el8.x86_64 2023-11-17T05:18:36.960 INFO:teuthology.orchestra.run.smithi040.stdout: python3-scipy-1.0.0-21.module_el8.5.0+771+e5d9a225.x86_64 2023-11-17T05:18:36.960 INFO:teuthology.orchestra.run.smithi040.stdout: 2023-11-17T05:18:36.960 INFO:teuthology.orchestra.run.smithi040.stdout:Complete! 2023-11-17T05:18:37.079 DEBUG:teuthology.orchestra.run.smithi040:> sudo yum -y install ceph-mgr-rook 2023-11-17T05:18:37.246 INFO:teuthology.orchestra.run.smithi066.stdout: Running scriptlet: ceph-mgr-rook-2:16.2.4-0.el8.noarch 13/13 2023-11-17T05:18:37.247 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : ceph-mgr-rook-2:16.2.4-0.el8.noarch 1/13 2023-11-17T05:18:37.247 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : python3-jsonpatch-1.21-2.el8.noarch 2/13 2023-11-17T05:18:37.247 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : python3-jsonpointer-1.10-11.el8.noarch 3/13 2023-11-17T05:18:37.247 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : python3-pyasn1-0.3.7-6.el8.noarch 4/13 2023-11-17T05:18:37.247 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : python3-pyasn1-modules-0.3.7-6.el8.noarch 5/13 2023-11-17T05:18:37.247 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : python3-oauthlib-2.1.0-1.el8.noarch 6/13 2023-11-17T05:18:37.247 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : python3-requests-oauthlib-1.0.0-1.el8.noarch 7/13 2023-11-17T05:18:37.248 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : python3-cachetools-3.1.1-4.el8.noarch 8/13 2023-11-17T05:18:37.248 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : python3-certifi-2018.10.15-7.el8.noarch 9/13 2023-11-17T05:18:37.248 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : python3-google-auth-1:1.1.1-10.el8.noarch 10/13 2023-11-17T05:18:37.248 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : python3-kubernetes-1:11.0.0-6.el8.noarch 11/13 2023-11-17T05:18:37.248 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : python3-rsa-4.9-2.el8.noarch 12/13 2023-11-17T05:18:38.028 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : python3-websocket-client-0.56.0-5.el8.noarch 13/13 2023-11-17T05:18:38.028 INFO:teuthology.orchestra.run.smithi066.stdout: 2023-11-17T05:18:38.028 INFO:teuthology.orchestra.run.smithi066.stdout:Installed: 2023-11-17T05:18:38.029 INFO:teuthology.orchestra.run.smithi066.stdout: ceph-mgr-rook-2:16.2.4-0.el8.noarch 2023-11-17T05:18:38.029 INFO:teuthology.orchestra.run.smithi066.stdout: python3-cachetools-3.1.1-4.el8.noarch 2023-11-17T05:18:38.029 INFO:teuthology.orchestra.run.smithi066.stdout: python3-certifi-2018.10.15-7.el8.noarch 2023-11-17T05:18:38.029 INFO:teuthology.orchestra.run.smithi066.stdout: python3-google-auth-1:1.1.1-10.el8.noarch 2023-11-17T05:18:38.030 INFO:teuthology.orchestra.run.smithi066.stdout: python3-jsonpatch-1.21-2.el8.noarch 2023-11-17T05:18:38.030 INFO:teuthology.orchestra.run.smithi066.stdout: python3-jsonpointer-1.10-11.el8.noarch 2023-11-17T05:18:38.030 INFO:teuthology.orchestra.run.smithi066.stdout: python3-kubernetes-1:11.0.0-6.el8.noarch 2023-11-17T05:18:38.030 INFO:teuthology.orchestra.run.smithi066.stdout: python3-oauthlib-2.1.0-1.el8.noarch 2023-11-17T05:18:38.030 INFO:teuthology.orchestra.run.smithi066.stdout: python3-pyasn1-0.3.7-6.el8.noarch 2023-11-17T05:18:38.030 INFO:teuthology.orchestra.run.smithi066.stdout: python3-pyasn1-modules-0.3.7-6.el8.noarch 2023-11-17T05:18:38.030 INFO:teuthology.orchestra.run.smithi066.stdout: python3-requests-oauthlib-1.0.0-1.el8.noarch 2023-11-17T05:18:38.030 INFO:teuthology.orchestra.run.smithi066.stdout: python3-rsa-4.9-2.el8.noarch 2023-11-17T05:18:38.030 INFO:teuthology.orchestra.run.smithi066.stdout: python3-websocket-client-0.56.0-5.el8.noarch 2023-11-17T05:18:38.030 INFO:teuthology.orchestra.run.smithi066.stdout: 2023-11-17T05:18:38.030 INFO:teuthology.orchestra.run.smithi066.stdout:Complete! 2023-11-17T05:18:38.071 INFO:teuthology.orchestra.run.smithi040.stdout:Last metadata expiration check: 0:03:20 ago on Fri 17 Nov 2023 05:15:18 AM UTC. 2023-11-17T05:18:38.155 DEBUG:teuthology.orchestra.run.smithi066:> sudo yum -y install ceph-mgr-cephadm 2023-11-17T05:18:39.043 INFO:teuthology.orchestra.run.smithi066.stdout:Last metadata expiration check: 0:03:26 ago on Fri 17 Nov 2023 05:15:13 AM UTC. 2023-11-17T05:18:39.916 INFO:teuthology.orchestra.run.smithi040.stdout:Dependencies resolved. 2023-11-17T05:18:39.918 INFO:teuthology.orchestra.run.smithi040.stdout:================================================================================ 2023-11-17T05:18:39.918 INFO:teuthology.orchestra.run.smithi040.stdout: Package Arch Version Repository Size 2023-11-17T05:18:39.918 INFO:teuthology.orchestra.run.smithi040.stdout:================================================================================ 2023-11-17T05:18:39.918 INFO:teuthology.orchestra.run.smithi040.stdout:Installing: 2023-11-17T05:18:39.918 INFO:teuthology.orchestra.run.smithi040.stdout: ceph-mgr-rook noarch 2:16.2.4-0.el8 ceph-noarch 27 k 2023-11-17T05:18:39.918 INFO:teuthology.orchestra.run.smithi040.stdout:Installing dependencies: 2023-11-17T05:18:39.918 INFO:teuthology.orchestra.run.smithi040.stdout: python3-cachetools noarch 3.1.1-4.el8 epel 33 k 2023-11-17T05:18:39.918 INFO:teuthology.orchestra.run.smithi040.stdout: python3-certifi noarch 2018.10.15-7.el8 epel 16 k 2023-11-17T05:18:39.919 INFO:teuthology.orchestra.run.smithi040.stdout: python3-google-auth noarch 1:1.1.1-10.el8 epel 81 k 2023-11-17T05:18:39.919 INFO:teuthology.orchestra.run.smithi040.stdout: python3-jsonpatch noarch 1.21-2.el8 appstream 27 k 2023-11-17T05:18:39.919 INFO:teuthology.orchestra.run.smithi040.stdout: python3-jsonpointer noarch 1.10-11.el8 appstream 20 k 2023-11-17T05:18:39.919 INFO:teuthology.orchestra.run.smithi040.stdout: python3-kubernetes noarch 1:11.0.0-6.el8 epel 1.2 M 2023-11-17T05:18:39.919 INFO:teuthology.orchestra.run.smithi040.stdout: python3-oauthlib noarch 2.1.0-1.el8 baseos 155 k 2023-11-17T05:18:39.919 INFO:teuthology.orchestra.run.smithi040.stdout: python3-pyasn1 noarch 0.3.7-6.el8 appstream 126 k 2023-11-17T05:18:39.919 INFO:teuthology.orchestra.run.smithi040.stdout: python3-pyasn1-modules noarch 0.3.7-6.el8 appstream 110 k 2023-11-17T05:18:39.919 INFO:teuthology.orchestra.run.smithi040.stdout: python3-requests-oauthlib noarch 1.0.0-1.el8 baseos 43 k 2023-11-17T05:18:39.919 INFO:teuthology.orchestra.run.smithi040.stdout: python3-rsa noarch 4.9-2.el8 epel 64 k 2023-11-17T05:18:39.919 INFO:teuthology.orchestra.run.smithi040.stdout: python3-websocket-client noarch 0.56.0-5.el8 epel 61 k 2023-11-17T05:18:39.919 INFO:teuthology.orchestra.run.smithi040.stdout: 2023-11-17T05:18:39.919 INFO:teuthology.orchestra.run.smithi040.stdout:Transaction Summary 2023-11-17T05:18:39.919 INFO:teuthology.orchestra.run.smithi040.stdout:================================================================================ 2023-11-17T05:18:39.920 INFO:teuthology.orchestra.run.smithi040.stdout:Install 13 Packages 2023-11-17T05:18:39.920 INFO:teuthology.orchestra.run.smithi040.stdout: 2023-11-17T05:18:39.920 INFO:teuthology.orchestra.run.smithi040.stdout:Total download size: 1.9 M 2023-11-17T05:18:39.920 INFO:teuthology.orchestra.run.smithi040.stdout:Installed size: 24 M 2023-11-17T05:18:39.920 INFO:teuthology.orchestra.run.smithi040.stdout:Downloading Packages: 2023-11-17T05:18:40.188 INFO:teuthology.orchestra.run.smithi040.stdout:(1/13): ceph-mgr-rook-16.2.4-0.el8.noarch.rpm 559 kB/s | 27 kB 00:00 2023-11-17T05:18:40.272 INFO:teuthology.orchestra.run.smithi040.stdout:(2/13): python3-jsonpointer-1.10-11.el8.noarch. 152 kB/s | 20 kB 00:00 2023-11-17T05:18:40.297 INFO:teuthology.orchestra.run.smithi040.stdout:(3/13): python3-jsonpatch-1.21-2.el8.noarch.rpm 169 kB/s | 27 kB 00:00 2023-11-17T05:18:40.380 INFO:teuthology.orchestra.run.smithi040.stdout:(4/13): python3-pyasn1-0.3.7-6.el8.noarch.rpm 658 kB/s | 126 kB 00:00 2023-11-17T05:18:40.398 INFO:teuthology.orchestra.run.smithi040.stdout:(5/13): python3-pyasn1-modules-0.3.7-6.el8.noar 875 kB/s | 110 kB 00:00 2023-11-17T05:18:40.429 INFO:teuthology.orchestra.run.smithi040.stdout:(6/13): python3-oauthlib-2.1.0-1.el8.noarch.rpm 1.2 MB/s | 155 kB 00:00 2023-11-17T05:18:40.466 INFO:teuthology.orchestra.run.smithi040.stdout:(7/13): python3-requests-oauthlib-1.0.0-1.el8.n 507 kB/s | 43 kB 00:00 2023-11-17T05:18:40.565 INFO:teuthology.orchestra.run.smithi040.stdout:(8/13): python3-cachetools-3.1.1-4.el8.noarch.r 197 kB/s | 33 kB 00:00 2023-11-17T05:18:40.583 INFO:teuthology.orchestra.run.smithi040.stdout:(9/13): python3-certifi-2018.10.15-7.el8.noarch 103 kB/s | 16 kB 00:00 2023-11-17T05:18:40.641 INFO:teuthology.orchestra.run.smithi040.stdout:(10/13): python3-google-auth-1.1.1-10.el8.noarc 464 kB/s | 81 kB 00:00 2023-11-17T05:18:40.682 INFO:teuthology.orchestra.run.smithi040.stdout:(11/13): python3-rsa-4.9-2.el8.noarch.rpm 642 kB/s | 64 kB 00:00 2023-11-17T05:18:40.699 INFO:teuthology.orchestra.run.smithi040.stdout:(12/13): python3-websocket-client-0.56.0-5.el8. 1.0 MB/s | 61 kB 00:00 2023-11-17T05:18:40.817 INFO:teuthology.orchestra.run.smithi040.stdout:(13/13): python3-kubernetes-11.0.0-6.el8.noarch 5.0 MB/s | 1.2 MB 00:00 2023-11-17T05:18:40.817 INFO:teuthology.orchestra.run.smithi040.stdout:-------------------------------------------------------------------------------- 2023-11-17T05:18:40.817 INFO:teuthology.orchestra.run.smithi040.stdout:Total 2.1 MB/s | 1.9 MB 00:00 2023-11-17T05:18:40.819 INFO:teuthology.orchestra.run.smithi066.stdout:Dependencies resolved. 2023-11-17T05:18:40.820 INFO:teuthology.orchestra.run.smithi066.stdout:================================================================================ 2023-11-17T05:18:40.820 INFO:teuthology.orchestra.run.smithi066.stdout: Package Arch Version Repository Size 2023-11-17T05:18:40.820 INFO:teuthology.orchestra.run.smithi066.stdout:================================================================================ 2023-11-17T05:18:40.820 INFO:teuthology.orchestra.run.smithi066.stdout:Installing: 2023-11-17T05:18:40.820 INFO:teuthology.orchestra.run.smithi066.stdout: ceph-mgr-cephadm noarch 2:16.2.4-0.el8 ceph-noarch 89 k 2023-11-17T05:18:40.820 INFO:teuthology.orchestra.run.smithi066.stdout:Installing dependencies: 2023-11-17T05:18:40.821 INFO:teuthology.orchestra.run.smithi066.stdout: python3-apipkg noarch 1.5-6.el8 epel 21 k 2023-11-17T05:18:40.821 INFO:teuthology.orchestra.run.smithi066.stdout: python3-babel noarch 2.5.1-7.el8 appstream 4.8 M 2023-11-17T05:18:40.821 INFO:teuthology.orchestra.run.smithi066.stdout: python3-execnet noarch 1.7.1-1.el8 epel 355 k 2023-11-17T05:18:40.821 INFO:teuthology.orchestra.run.smithi066.stdout: python3-jinja2 noarch 2.10.1-3.el8 appstream 538 k 2023-11-17T05:18:40.821 INFO:teuthology.orchestra.run.smithi066.stdout: python3-remoto noarch 1.2.1-1.el8 epel 49 k 2023-11-17T05:18:40.821 INFO:teuthology.orchestra.run.smithi066.stdout: 2023-11-17T05:18:40.821 INFO:teuthology.orchestra.run.smithi066.stdout:Transaction Summary 2023-11-17T05:18:40.821 INFO:teuthology.orchestra.run.smithi066.stdout:================================================================================ 2023-11-17T05:18:40.821 INFO:teuthology.orchestra.run.smithi066.stdout:Install 6 Packages 2023-11-17T05:18:40.821 INFO:teuthology.orchestra.run.smithi066.stdout: 2023-11-17T05:18:40.822 INFO:teuthology.orchestra.run.smithi066.stdout:Total download size: 5.8 M 2023-11-17T05:18:40.822 INFO:teuthology.orchestra.run.smithi066.stdout:Installed size: 24 M 2023-11-17T05:18:40.822 INFO:teuthology.orchestra.run.smithi066.stdout:Downloading Packages: 2023-11-17T05:18:40.881 INFO:teuthology.orchestra.run.smithi040.stdout:Running transaction check 2023-11-17T05:18:40.899 INFO:teuthology.orchestra.run.smithi040.stdout:Transaction check succeeded. 2023-11-17T05:18:40.899 INFO:teuthology.orchestra.run.smithi040.stdout:Running transaction test 2023-11-17T05:18:40.998 INFO:teuthology.orchestra.run.smithi040.stdout:Transaction test succeeded. 2023-11-17T05:18:41.001 INFO:teuthology.orchestra.run.smithi040.stdout:Running transaction 2023-11-17T05:18:41.202 INFO:teuthology.orchestra.run.smithi066.stdout:(1/6): ceph-mgr-cephadm-16.2.4-0.el8.noarch.rpm 1.8 MB/s | 89 kB 00:00 2023-11-17T05:18:41.430 INFO:teuthology.orchestra.run.smithi066.stdout:(2/6): python3-jinja2-2.10.1-3.el8.noarch.rpm 1.9 MB/s | 538 kB 00:00 2023-11-17T05:18:41.528 INFO:teuthology.orchestra.run.smithi040.stdout: Preparing : 1/1 2023-11-17T05:18:41.553 INFO:teuthology.orchestra.run.smithi066.stdout:(3/6): python3-apipkg-1.5-6.el8.noarch.rpm 60 kB/s | 21 kB 00:00 2023-11-17T05:18:41.653 INFO:teuthology.orchestra.run.smithi066.stdout:(4/6): python3-babel-2.5.1-7.el8.noarch.rpm 9.6 MB/s | 4.8 MB 00:00 2023-11-17T05:18:41.728 INFO:teuthology.orchestra.run.smithi066.stdout:(5/6): python3-remoto-1.2.1-1.el8.noarch.rpm 279 kB/s | 49 kB 00:00 2023-11-17T05:18:41.784 INFO:teuthology.orchestra.run.smithi040.stdout: Installing : python3-pyasn1-0.3.7-6.el8.noarch 1/13 2023-11-17T05:18:41.954 INFO:teuthology.orchestra.run.smithi066.stdout:(6/6): python3-execnet-1.7.1-1.el8.noarch.rpm 679 kB/s | 355 kB 00:00 2023-11-17T05:18:41.954 INFO:teuthology.orchestra.run.smithi066.stdout:-------------------------------------------------------------------------------- 2023-11-17T05:18:41.954 INFO:teuthology.orchestra.run.smithi066.stdout:Total 5.1 MB/s | 5.8 MB 00:01 2023-11-17T05:18:41.978 INFO:teuthology.orchestra.run.smithi040.stdout: Installing : python3-pyasn1-modules-0.3.7-6.el8.noarch 2/13 2023-11-17T05:18:42.008 INFO:teuthology.orchestra.run.smithi066.stdout:Running transaction check 2023-11-17T05:18:42.019 INFO:teuthology.orchestra.run.smithi066.stdout:Transaction check succeeded. 2023-11-17T05:18:42.019 INFO:teuthology.orchestra.run.smithi066.stdout:Running transaction test 2023-11-17T05:18:42.129 INFO:teuthology.orchestra.run.smithi066.stdout:Transaction test succeeded. 2023-11-17T05:18:42.132 INFO:teuthology.orchestra.run.smithi066.stdout:Running transaction 2023-11-17T05:18:42.206 INFO:teuthology.orchestra.run.smithi040.stdout: Installing : python3-rsa-4.9-2.el8.noarch 3/13 2023-11-17T05:18:42.414 INFO:teuthology.orchestra.run.smithi066.stdout: Preparing : 1/1 2023-11-17T05:18:42.491 INFO:teuthology.orchestra.run.smithi040.stdout: Installing : python3-websocket-client-0.56.0-5.el8.noarch 4/13 2023-11-17T05:18:42.667 INFO:teuthology.orchestra.run.smithi066.stdout: Installing : python3-apipkg-1.5-6.el8.noarch 1/6 2023-11-17T05:18:42.705 INFO:teuthology.orchestra.run.smithi040.stdout: Installing : python3-certifi-2018.10.15-7.el8.noarch 5/13 2023-11-17T05:18:42.960 INFO:teuthology.orchestra.run.smithi040.stdout: Installing : python3-cachetools-3.1.1-4.el8.noarch 6/13 2023-11-17T05:18:42.985 INFO:teuthology.orchestra.run.smithi066.stdout: Installing : python3-execnet-1.7.1-1.el8.noarch 2/6 2023-11-17T05:18:43.296 INFO:teuthology.orchestra.run.smithi040.stdout: Installing : python3-google-auth-1:1.1.1-10.el8.noarch 7/13 2023-11-17T05:18:43.524 INFO:teuthology.orchestra.run.smithi040.stdout: Installing : python3-oauthlib-2.1.0-1.el8.noarch 8/13 2023-11-17T05:18:43.958 INFO:teuthology.orchestra.run.smithi066.stdout: Installing : python3-remoto-1.2.1-1.el8.noarch 3/6 2023-11-17T05:18:44.317 INFO:teuthology.orchestra.run.smithi066.stdout: Installing : python3-babel-2.5.1-7.el8.noarch 4/6 2023-11-17T05:18:44.569 INFO:teuthology.orchestra.run.smithi066.stdout: Installing : python3-jinja2-2.10.1-3.el8.noarch 5/6 2023-11-17T05:18:44.682 INFO:teuthology.orchestra.run.smithi040.stdout: Installing : python3-requests-oauthlib-1.0.0-1.el8.noarch 9/13 2023-11-17T05:18:44.754 INFO:teuthology.orchestra.run.smithi066.stdout: Installing : ceph-mgr-cephadm-2:16.2.4-0.el8.noarch 6/6 2023-11-17T05:18:45.007 INFO:teuthology.orchestra.run.smithi040.stdout: Installing : python3-kubernetes-1:11.0.0-6.el8.noarch 10/13 2023-11-17T05:18:45.038 INFO:teuthology.orchestra.run.smithi066.stdout: Running scriptlet: ceph-mgr-cephadm-2:16.2.4-0.el8.noarch 6/6 2023-11-17T05:18:45.038 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : ceph-mgr-cephadm-2:16.2.4-0.el8.noarch 1/6 2023-11-17T05:18:45.038 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : python3-babel-2.5.1-7.el8.noarch 2/6 2023-11-17T05:18:45.039 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : python3-jinja2-2.10.1-3.el8.noarch 3/6 2023-11-17T05:18:45.039 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : python3-apipkg-1.5-6.el8.noarch 4/6 2023-11-17T05:18:45.039 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : python3-execnet-1.7.1-1.el8.noarch 5/6 2023-11-17T05:18:45.199 INFO:teuthology.orchestra.run.smithi040.stdout: Installing : python3-jsonpointer-1.10-11.el8.noarch 11/13 2023-11-17T05:18:45.391 INFO:teuthology.orchestra.run.smithi040.stdout: Installing : python3-jsonpatch-1.21-2.el8.noarch 12/13 2023-11-17T05:18:45.556 INFO:teuthology.orchestra.run.smithi040.stdout: Installing : ceph-mgr-rook-2:16.2.4-0.el8.noarch 13/13 2023-11-17T05:18:45.905 INFO:teuthology.orchestra.run.smithi040.stdout: Running scriptlet: ceph-mgr-rook-2:16.2.4-0.el8.noarch 13/13 2023-11-17T05:18:45.905 INFO:teuthology.orchestra.run.smithi040.stdout: Verifying : ceph-mgr-rook-2:16.2.4-0.el8.noarch 1/13 2023-11-17T05:18:45.906 INFO:teuthology.orchestra.run.smithi040.stdout: Verifying : python3-jsonpatch-1.21-2.el8.noarch 2/13 2023-11-17T05:18:45.906 INFO:teuthology.orchestra.run.smithi040.stdout: Verifying : python3-jsonpointer-1.10-11.el8.noarch 3/13 2023-11-17T05:18:45.906 INFO:teuthology.orchestra.run.smithi040.stdout: Verifying : python3-pyasn1-0.3.7-6.el8.noarch 4/13 2023-11-17T05:18:45.906 INFO:teuthology.orchestra.run.smithi040.stdout: Verifying : python3-pyasn1-modules-0.3.7-6.el8.noarch 5/13 2023-11-17T05:18:45.906 INFO:teuthology.orchestra.run.smithi040.stdout: Verifying : python3-oauthlib-2.1.0-1.el8.noarch 6/13 2023-11-17T05:18:45.906 INFO:teuthology.orchestra.run.smithi040.stdout: Verifying : python3-requests-oauthlib-1.0.0-1.el8.noarch 7/13 2023-11-17T05:18:45.906 INFO:teuthology.orchestra.run.smithi040.stdout: Verifying : python3-cachetools-3.1.1-4.el8.noarch 8/13 2023-11-17T05:18:45.906 INFO:teuthology.orchestra.run.smithi040.stdout: Verifying : python3-certifi-2018.10.15-7.el8.noarch 9/13 2023-11-17T05:18:45.906 INFO:teuthology.orchestra.run.smithi040.stdout: Verifying : python3-google-auth-1:1.1.1-10.el8.noarch 10/13 2023-11-17T05:18:45.906 INFO:teuthology.orchestra.run.smithi040.stdout: Verifying : python3-kubernetes-1:11.0.0-6.el8.noarch 11/13 2023-11-17T05:18:45.906 INFO:teuthology.orchestra.run.smithi040.stdout: Verifying : python3-rsa-4.9-2.el8.noarch 12/13 2023-11-17T05:18:45.926 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : python3-remoto-1.2.1-1.el8.noarch 6/6 2023-11-17T05:18:45.926 INFO:teuthology.orchestra.run.smithi066.stdout: 2023-11-17T05:18:45.926 INFO:teuthology.orchestra.run.smithi066.stdout:Installed: 2023-11-17T05:18:45.926 INFO:teuthology.orchestra.run.smithi066.stdout: ceph-mgr-cephadm-2:16.2.4-0.el8.noarch python3-apipkg-1.5-6.el8.noarch 2023-11-17T05:18:45.926 INFO:teuthology.orchestra.run.smithi066.stdout: python3-babel-2.5.1-7.el8.noarch python3-execnet-1.7.1-1.el8.noarch 2023-11-17T05:18:45.926 INFO:teuthology.orchestra.run.smithi066.stdout: python3-jinja2-2.10.1-3.el8.noarch python3-remoto-1.2.1-1.el8.noarch 2023-11-17T05:18:45.926 INFO:teuthology.orchestra.run.smithi066.stdout: 2023-11-17T05:18:45.926 INFO:teuthology.orchestra.run.smithi066.stdout:Complete! 2023-11-17T05:18:46.063 DEBUG:teuthology.orchestra.run.smithi066:> sudo yum -y install ceph-fuse 2023-11-17T05:18:46.723 INFO:teuthology.orchestra.run.smithi040.stdout: Verifying : python3-websocket-client-0.56.0-5.el8.noarch 13/13 2023-11-17T05:18:46.723 INFO:teuthology.orchestra.run.smithi040.stdout: 2023-11-17T05:18:46.723 INFO:teuthology.orchestra.run.smithi040.stdout:Installed: 2023-11-17T05:18:46.723 INFO:teuthology.orchestra.run.smithi040.stdout: ceph-mgr-rook-2:16.2.4-0.el8.noarch 2023-11-17T05:18:46.723 INFO:teuthology.orchestra.run.smithi040.stdout: python3-cachetools-3.1.1-4.el8.noarch 2023-11-17T05:18:46.723 INFO:teuthology.orchestra.run.smithi040.stdout: python3-certifi-2018.10.15-7.el8.noarch 2023-11-17T05:18:46.723 INFO:teuthology.orchestra.run.smithi040.stdout: python3-google-auth-1:1.1.1-10.el8.noarch 2023-11-17T05:18:46.723 INFO:teuthology.orchestra.run.smithi040.stdout: python3-jsonpatch-1.21-2.el8.noarch 2023-11-17T05:18:46.723 INFO:teuthology.orchestra.run.smithi040.stdout: python3-jsonpointer-1.10-11.el8.noarch 2023-11-17T05:18:46.724 INFO:teuthology.orchestra.run.smithi040.stdout: python3-kubernetes-1:11.0.0-6.el8.noarch 2023-11-17T05:18:46.724 INFO:teuthology.orchestra.run.smithi040.stdout: python3-oauthlib-2.1.0-1.el8.noarch 2023-11-17T05:18:46.724 INFO:teuthology.orchestra.run.smithi040.stdout: python3-pyasn1-0.3.7-6.el8.noarch 2023-11-17T05:18:46.724 INFO:teuthology.orchestra.run.smithi040.stdout: python3-pyasn1-modules-0.3.7-6.el8.noarch 2023-11-17T05:18:46.724 INFO:teuthology.orchestra.run.smithi040.stdout: python3-requests-oauthlib-1.0.0-1.el8.noarch 2023-11-17T05:18:46.724 INFO:teuthology.orchestra.run.smithi040.stdout: python3-rsa-4.9-2.el8.noarch 2023-11-17T05:18:46.724 INFO:teuthology.orchestra.run.smithi040.stdout: python3-websocket-client-0.56.0-5.el8.noarch 2023-11-17T05:18:46.724 INFO:teuthology.orchestra.run.smithi040.stdout: 2023-11-17T05:18:46.724 INFO:teuthology.orchestra.run.smithi040.stdout:Complete! 2023-11-17T05:18:46.863 DEBUG:teuthology.orchestra.run.smithi040:> sudo yum -y install ceph-mgr-cephadm 2023-11-17T05:18:46.954 INFO:teuthology.orchestra.run.smithi066.stdout:Last metadata expiration check: 0:03:33 ago on Fri 17 Nov 2023 05:15:13 AM UTC. 2023-11-17T05:18:47.856 INFO:teuthology.orchestra.run.smithi040.stdout:Last metadata expiration check: 0:03:29 ago on Fri 17 Nov 2023 05:15:18 AM UTC. 2023-11-17T05:18:48.729 INFO:teuthology.orchestra.run.smithi066.stdout:Dependencies resolved. 2023-11-17T05:18:48.730 INFO:teuthology.orchestra.run.smithi066.stdout:================================================================================ 2023-11-17T05:18:48.730 INFO:teuthology.orchestra.run.smithi066.stdout: Package Architecture Version Repository Size 2023-11-17T05:18:48.730 INFO:teuthology.orchestra.run.smithi066.stdout:================================================================================ 2023-11-17T05:18:48.730 INFO:teuthology.orchestra.run.smithi066.stdout:Installing: 2023-11-17T05:18:48.730 INFO:teuthology.orchestra.run.smithi066.stdout: ceph-fuse x86_64 2:16.2.4-0.el8 ceph 885 k 2023-11-17T05:18:48.730 INFO:teuthology.orchestra.run.smithi066.stdout:Installing dependencies: 2023-11-17T05:18:48.730 INFO:teuthology.orchestra.run.smithi066.stdout: fuse x86_64 2.9.7-17.el8 baseos 83 k 2023-11-17T05:18:48.730 INFO:teuthology.orchestra.run.smithi066.stdout: 2023-11-17T05:18:48.731 INFO:teuthology.orchestra.run.smithi066.stdout:Transaction Summary 2023-11-17T05:18:48.731 INFO:teuthology.orchestra.run.smithi066.stdout:================================================================================ 2023-11-17T05:18:48.731 INFO:teuthology.orchestra.run.smithi066.stdout:Install 2 Packages 2023-11-17T05:18:48.731 INFO:teuthology.orchestra.run.smithi066.stdout: 2023-11-17T05:18:48.731 INFO:teuthology.orchestra.run.smithi066.stdout:Total download size: 968 k 2023-11-17T05:18:48.731 INFO:teuthology.orchestra.run.smithi066.stdout:Installed size: 3.2 M 2023-11-17T05:18:48.731 INFO:teuthology.orchestra.run.smithi066.stdout:Downloading Packages: 2023-11-17T05:18:48.989 INFO:teuthology.orchestra.run.smithi066.stdout:(1/2): ceph-fuse-16.2.4-0.el8.x86_64.rpm 13 MB/s | 885 kB 00:00 2023-11-17T05:18:49.067 INFO:teuthology.orchestra.run.smithi066.stdout:(2/2): fuse-2.9.7-17.el8.x86_64.rpm 588 kB/s | 83 kB 00:00 2023-11-17T05:18:49.067 INFO:teuthology.orchestra.run.smithi066.stdout:-------------------------------------------------------------------------------- 2023-11-17T05:18:49.067 INFO:teuthology.orchestra.run.smithi066.stdout:Total 2.8 MB/s | 968 kB 00:00 2023-11-17T05:18:49.076 INFO:teuthology.orchestra.run.smithi066.stdout:Running transaction check 2023-11-17T05:18:49.097 INFO:teuthology.orchestra.run.smithi066.stdout:Transaction check succeeded. 2023-11-17T05:18:49.097 INFO:teuthology.orchestra.run.smithi066.stdout:Running transaction test 2023-11-17T05:18:49.190 INFO:teuthology.orchestra.run.smithi066.stdout:Transaction test succeeded. 2023-11-17T05:18:49.193 INFO:teuthology.orchestra.run.smithi066.stdout:Running transaction 2023-11-17T05:18:49.370 INFO:teuthology.orchestra.run.smithi066.stdout: Preparing : 1/1 2023-11-17T05:18:49.626 INFO:teuthology.orchestra.run.smithi066.stdout: Installing : fuse-2.9.7-17.el8.x86_64 1/2 2023-11-17T05:18:49.702 INFO:teuthology.orchestra.run.smithi040.stdout:Dependencies resolved. 2023-11-17T05:18:49.703 INFO:teuthology.orchestra.run.smithi040.stdout:================================================================================ 2023-11-17T05:18:49.703 INFO:teuthology.orchestra.run.smithi040.stdout: Package Arch Version Repository Size 2023-11-17T05:18:49.704 INFO:teuthology.orchestra.run.smithi040.stdout:================================================================================ 2023-11-17T05:18:49.704 INFO:teuthology.orchestra.run.smithi040.stdout:Installing: 2023-11-17T05:18:49.704 INFO:teuthology.orchestra.run.smithi040.stdout: ceph-mgr-cephadm noarch 2:16.2.4-0.el8 ceph-noarch 89 k 2023-11-17T05:18:49.704 INFO:teuthology.orchestra.run.smithi040.stdout:Installing dependencies: 2023-11-17T05:18:49.704 INFO:teuthology.orchestra.run.smithi040.stdout: python3-apipkg noarch 1.5-6.el8 epel 21 k 2023-11-17T05:18:49.705 INFO:teuthology.orchestra.run.smithi040.stdout: python3-babel noarch 2.5.1-7.el8 appstream 4.8 M 2023-11-17T05:18:49.705 INFO:teuthology.orchestra.run.smithi040.stdout: python3-execnet noarch 1.7.1-1.el8 epel 355 k 2023-11-17T05:18:49.705 INFO:teuthology.orchestra.run.smithi040.stdout: python3-jinja2 noarch 2.10.1-3.el8 appstream 538 k 2023-11-17T05:18:49.705 INFO:teuthology.orchestra.run.smithi040.stdout: python3-remoto noarch 1.2.1-1.el8 epel 49 k 2023-11-17T05:18:49.705 INFO:teuthology.orchestra.run.smithi040.stdout: 2023-11-17T05:18:49.706 INFO:teuthology.orchestra.run.smithi040.stdout:Transaction Summary 2023-11-17T05:18:49.706 INFO:teuthology.orchestra.run.smithi040.stdout:================================================================================ 2023-11-17T05:18:49.706 INFO:teuthology.orchestra.run.smithi040.stdout:Install 6 Packages 2023-11-17T05:18:49.706 INFO:teuthology.orchestra.run.smithi040.stdout: 2023-11-17T05:18:49.707 INFO:teuthology.orchestra.run.smithi040.stdout:Total download size: 5.8 M 2023-11-17T05:18:49.707 INFO:teuthology.orchestra.run.smithi040.stdout:Installed size: 24 M 2023-11-17T05:18:49.707 INFO:teuthology.orchestra.run.smithi040.stdout:Downloading Packages: 2023-11-17T05:18:49.838 INFO:teuthology.orchestra.run.smithi066.stdout: Installing : ceph-fuse-2:16.2.4-0.el8.x86_64 2/2 2023-11-17T05:18:50.255 INFO:teuthology.orchestra.run.smithi066.stdout: Running scriptlet: ceph-fuse-2:16.2.4-0.el8.x86_64 2/2 2023-11-17T05:18:50.255 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : ceph-fuse-2:16.2.4-0.el8.x86_64 1/2 2023-11-17T05:18:50.282 INFO:teuthology.orchestra.run.smithi040.stdout:(1/6): ceph-mgr-cephadm-16.2.4-0.el8.noarch.rpm 2.1 MB/s | 89 kB 00:00 2023-11-17T05:18:50.499 INFO:teuthology.orchestra.run.smithi040.stdout:(2/6): python3-jinja2-2.10.1-3.el8.noarch.rpm 2.0 MB/s | 538 kB 00:00 2023-11-17T05:18:50.749 INFO:teuthology.orchestra.run.smithi040.stdout:(3/6): python3-apipkg-1.5-6.el8.noarch.rpm 45 kB/s | 21 kB 00:00 2023-11-17T05:18:50.842 INFO:teuthology.orchestra.run.smithi040.stdout:(4/6): python3-babel-2.5.1-7.el8.noarch.rpm 8.0 MB/s | 4.8 MB 00:00 2023-11-17T05:18:50.942 INFO:teuthology.orchestra.run.smithi040.stdout:(5/6): python3-remoto-1.2.1-1.el8.noarch.rpm 254 kB/s | 49 kB 00:00 2023-11-17T05:18:51.021 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : fuse-2.9.7-17.el8.x86_64 2/2 2023-11-17T05:18:51.022 INFO:teuthology.orchestra.run.smithi066.stdout: 2023-11-17T05:18:51.022 INFO:teuthology.orchestra.run.smithi066.stdout:Installed: 2023-11-17T05:18:51.022 INFO:teuthology.orchestra.run.smithi066.stdout: ceph-fuse-2:16.2.4-0.el8.x86_64 fuse-2.9.7-17.el8.x86_64 2023-11-17T05:18:51.022 INFO:teuthology.orchestra.run.smithi066.stdout: 2023-11-17T05:18:51.022 INFO:teuthology.orchestra.run.smithi066.stdout:Complete! 2023-11-17T05:18:51.425 INFO:teuthology.orchestra.run.smithi040.stdout:(6/6): python3-execnet-1.7.1-1.el8.noarch.rpm 384 kB/s | 355 kB 00:00 2023-11-17T05:18:51.427 INFO:teuthology.orchestra.run.smithi040.stdout:-------------------------------------------------------------------------------- 2023-11-17T05:18:51.428 INFO:teuthology.orchestra.run.smithi040.stdout:Total 3.4 MB/s | 5.8 MB 00:01 2023-11-17T05:18:51.485 INFO:teuthology.orchestra.run.smithi040.stdout:Running transaction check 2023-11-17T05:18:51.495 INFO:teuthology.orchestra.run.smithi040.stdout:Transaction check succeeded. 2023-11-17T05:18:51.496 INFO:teuthology.orchestra.run.smithi040.stdout:Running transaction test 2023-11-17T05:18:51.618 INFO:teuthology.orchestra.run.smithi040.stdout:Transaction test succeeded. 2023-11-17T05:18:51.621 INFO:teuthology.orchestra.run.smithi040.stdout:Running transaction 2023-11-17T05:18:51.906 INFO:teuthology.orchestra.run.smithi040.stdout: Preparing : 1/1 2023-11-17T05:18:51.986 DEBUG:teuthology.orchestra.run.smithi066:> sudo yum -y install librados-devel 2023-11-17T05:18:52.218 INFO:teuthology.orchestra.run.smithi040.stdout: Installing : python3-apipkg-1.5-6.el8.noarch 1/6 2023-11-17T05:18:52.495 INFO:teuthology.orchestra.run.smithi040.stdout: Installing : python3-execnet-1.7.1-1.el8.noarch 2/6 2023-11-17T05:18:52.897 INFO:teuthology.orchestra.run.smithi066.stdout:Last metadata expiration check: 0:03:39 ago on Fri 17 Nov 2023 05:15:13 AM UTC. 2023-11-17T05:18:53.766 INFO:teuthology.orchestra.run.smithi040.stdout: Installing : python3-remoto-1.2.1-1.el8.noarch 3/6 2023-11-17T05:18:54.268 INFO:teuthology.orchestra.run.smithi040.stdout: Installing : python3-babel-2.5.1-7.el8.noarch 4/6 2023-11-17T05:18:54.480 INFO:teuthology.orchestra.run.smithi040.stdout: Installing : python3-jinja2-2.10.1-3.el8.noarch 5/6 2023-11-17T05:18:54.673 INFO:teuthology.orchestra.run.smithi066.stdout:Dependencies resolved. 2023-11-17T05:18:54.673 INFO:teuthology.orchestra.run.smithi066.stdout:================================================================================ 2023-11-17T05:18:54.673 INFO:teuthology.orchestra.run.smithi066.stdout: Package Architecture Version Repository Size 2023-11-17T05:18:54.673 INFO:teuthology.orchestra.run.smithi066.stdout:================================================================================ 2023-11-17T05:18:54.673 INFO:teuthology.orchestra.run.smithi066.stdout:Installing: 2023-11-17T05:18:54.674 INFO:teuthology.orchestra.run.smithi066.stdout: librados-devel x86_64 2:16.2.4-0.el8 ceph 131 k 2023-11-17T05:18:54.674 INFO:teuthology.orchestra.run.smithi066.stdout: 2023-11-17T05:18:54.674 INFO:teuthology.orchestra.run.smithi066.stdout:Transaction Summary 2023-11-17T05:18:54.674 INFO:teuthology.orchestra.run.smithi066.stdout:================================================================================ 2023-11-17T05:18:54.674 INFO:teuthology.orchestra.run.smithi066.stdout:Install 1 Package 2023-11-17T05:18:54.674 INFO:teuthology.orchestra.run.smithi066.stdout: 2023-11-17T05:18:54.674 INFO:teuthology.orchestra.run.smithi066.stdout:Total download size: 131 k 2023-11-17T05:18:54.674 INFO:teuthology.orchestra.run.smithi066.stdout:Installed size: 439 k 2023-11-17T05:18:54.674 INFO:teuthology.orchestra.run.smithi066.stdout:Downloading Packages: 2023-11-17T05:18:54.697 INFO:teuthology.orchestra.run.smithi040.stdout: Installing : ceph-mgr-cephadm-2:16.2.4-0.el8.noarch 6/6 2023-11-17T05:18:54.720 INFO:teuthology.orchestra.run.smithi066.stdout:librados-devel-16.2.4-0.el8.x86_64.rpm 2.8 MB/s | 131 kB 00:00 2023-11-17T05:18:54.720 INFO:teuthology.orchestra.run.smithi066.stdout:-------------------------------------------------------------------------------- 2023-11-17T05:18:54.720 INFO:teuthology.orchestra.run.smithi066.stdout:Total 2.7 MB/s | 131 kB 00:00 2023-11-17T05:18:54.720 INFO:teuthology.orchestra.run.smithi066.stdout:Running transaction check 2023-11-17T05:18:54.726 INFO:teuthology.orchestra.run.smithi066.stdout:Transaction check succeeded. 2023-11-17T05:18:54.726 INFO:teuthology.orchestra.run.smithi066.stdout:Running transaction test 2023-11-17T05:18:54.779 INFO:teuthology.orchestra.run.smithi066.stdout:Transaction test succeeded. 2023-11-17T05:18:54.782 INFO:teuthology.orchestra.run.smithi066.stdout:Running transaction 2023-11-17T05:18:54.893 INFO:teuthology.orchestra.run.smithi066.stdout: Preparing : 1/1 2023-11-17T05:18:55.015 INFO:teuthology.orchestra.run.smithi040.stdout: Running scriptlet: ceph-mgr-cephadm-2:16.2.4-0.el8.noarch 6/6 2023-11-17T05:18:55.015 INFO:teuthology.orchestra.run.smithi040.stdout: Verifying : ceph-mgr-cephadm-2:16.2.4-0.el8.noarch 1/6 2023-11-17T05:18:55.015 INFO:teuthology.orchestra.run.smithi040.stdout: Verifying : python3-babel-2.5.1-7.el8.noarch 2/6 2023-11-17T05:18:55.015 INFO:teuthology.orchestra.run.smithi040.stdout: Verifying : python3-jinja2-2.10.1-3.el8.noarch 3/6 2023-11-17T05:18:55.016 INFO:teuthology.orchestra.run.smithi040.stdout: Verifying : python3-apipkg-1.5-6.el8.noarch 4/6 2023-11-17T05:18:55.016 INFO:teuthology.orchestra.run.smithi040.stdout: Verifying : python3-execnet-1.7.1-1.el8.noarch 5/6 2023-11-17T05:18:55.103 INFO:teuthology.orchestra.run.smithi066.stdout: Installing : librados-devel-2:16.2.4-0.el8.x86_64 1/1 2023-11-17T05:18:55.417 INFO:teuthology.orchestra.run.smithi066.stdout: Running scriptlet: librados-devel-2:16.2.4-0.el8.x86_64 1/1 2023-11-17T05:18:55.780 INFO:teuthology.orchestra.run.smithi040.stdout: Verifying : python3-remoto-1.2.1-1.el8.noarch 6/6 2023-11-17T05:18:55.780 INFO:teuthology.orchestra.run.smithi040.stdout: 2023-11-17T05:18:55.781 INFO:teuthology.orchestra.run.smithi040.stdout:Installed: 2023-11-17T05:18:55.781 INFO:teuthology.orchestra.run.smithi040.stdout: ceph-mgr-cephadm-2:16.2.4-0.el8.noarch python3-apipkg-1.5-6.el8.noarch 2023-11-17T05:18:55.781 INFO:teuthology.orchestra.run.smithi040.stdout: python3-babel-2.5.1-7.el8.noarch python3-execnet-1.7.1-1.el8.noarch 2023-11-17T05:18:55.781 INFO:teuthology.orchestra.run.smithi040.stdout: python3-jinja2-2.10.1-3.el8.noarch python3-remoto-1.2.1-1.el8.noarch 2023-11-17T05:18:55.781 INFO:teuthology.orchestra.run.smithi040.stdout: 2023-11-17T05:18:55.781 INFO:teuthology.orchestra.run.smithi040.stdout:Complete! 2023-11-17T05:18:55.936 DEBUG:teuthology.orchestra.run.smithi040:> sudo yum -y install ceph-fuse 2023-11-17T05:18:56.116 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : librados-devel-2:16.2.4-0.el8.x86_64 1/1 2023-11-17T05:18:56.116 INFO:teuthology.orchestra.run.smithi066.stdout: 2023-11-17T05:18:56.116 INFO:teuthology.orchestra.run.smithi066.stdout:Installed: 2023-11-17T05:18:56.116 INFO:teuthology.orchestra.run.smithi066.stdout: librados-devel-2:16.2.4-0.el8.x86_64 2023-11-17T05:18:56.116 INFO:teuthology.orchestra.run.smithi066.stdout: 2023-11-17T05:18:56.116 INFO:teuthology.orchestra.run.smithi066.stdout:Complete! 2023-11-17T05:18:56.301 DEBUG:teuthology.orchestra.run.smithi066:> sudo yum -y install libcephfs2 2023-11-17T05:18:56.924 INFO:teuthology.orchestra.run.smithi040.stdout:Last metadata expiration check: 0:03:38 ago on Fri 17 Nov 2023 05:15:18 AM UTC. 2023-11-17T05:18:57.217 INFO:teuthology.orchestra.run.smithi066.stdout:Last metadata expiration check: 0:03:44 ago on Fri 17 Nov 2023 05:15:13 AM UTC. 2023-11-17T05:18:58.771 INFO:teuthology.orchestra.run.smithi066.stdout:Package libcephfs2-2:16.2.4-0.el8.x86_64 is already installed. 2023-11-17T05:18:58.791 INFO:teuthology.orchestra.run.smithi040.stdout:Dependencies resolved. 2023-11-17T05:18:58.792 INFO:teuthology.orchestra.run.smithi040.stdout:================================================================================ 2023-11-17T05:18:58.792 INFO:teuthology.orchestra.run.smithi040.stdout: Package Architecture Version Repository Size 2023-11-17T05:18:58.792 INFO:teuthology.orchestra.run.smithi040.stdout:================================================================================ 2023-11-17T05:18:58.792 INFO:teuthology.orchestra.run.smithi040.stdout:Installing: 2023-11-17T05:18:58.792 INFO:teuthology.orchestra.run.smithi040.stdout: ceph-fuse x86_64 2:16.2.4-0.el8 ceph 885 k 2023-11-17T05:18:58.792 INFO:teuthology.orchestra.run.smithi040.stdout:Installing dependencies: 2023-11-17T05:18:58.793 INFO:teuthology.orchestra.run.smithi040.stdout: fuse x86_64 2.9.7-17.el8 baseos 83 k 2023-11-17T05:18:58.793 INFO:teuthology.orchestra.run.smithi040.stdout: 2023-11-17T05:18:58.793 INFO:teuthology.orchestra.run.smithi040.stdout:Transaction Summary 2023-11-17T05:18:58.793 INFO:teuthology.orchestra.run.smithi040.stdout:================================================================================ 2023-11-17T05:18:58.793 INFO:teuthology.orchestra.run.smithi040.stdout:Install 2 Packages 2023-11-17T05:18:58.793 INFO:teuthology.orchestra.run.smithi040.stdout: 2023-11-17T05:18:58.793 INFO:teuthology.orchestra.run.smithi040.stdout:Total download size: 968 k 2023-11-17T05:18:58.794 INFO:teuthology.orchestra.run.smithi040.stdout:Installed size: 3.2 M 2023-11-17T05:18:58.794 INFO:teuthology.orchestra.run.smithi040.stdout:Downloading Packages: 2023-11-17T05:18:59.002 INFO:teuthology.orchestra.run.smithi066.stdout:Dependencies resolved. 2023-11-17T05:18:59.003 INFO:teuthology.orchestra.run.smithi066.stdout:Nothing to do. 2023-11-17T05:18:59.003 INFO:teuthology.orchestra.run.smithi066.stdout:Complete! 2023-11-17T05:18:59.045 INFO:teuthology.orchestra.run.smithi040.stdout:(1/2): ceph-fuse-16.2.4-0.el8.x86_64.rpm 15 MB/s | 885 kB 00:00 2023-11-17T05:18:59.104 DEBUG:teuthology.orchestra.run.smithi066:> sudo yum -y install libcephfs-devel 2023-11-17T05:18:59.120 INFO:teuthology.orchestra.run.smithi040.stdout:(2/2): fuse-2.9.7-17.el8.x86_64.rpm 625 kB/s | 83 kB 00:00 2023-11-17T05:18:59.121 INFO:teuthology.orchestra.run.smithi040.stdout:-------------------------------------------------------------------------------- 2023-11-17T05:18:59.121 INFO:teuthology.orchestra.run.smithi040.stdout:Total 2.9 MB/s | 968 kB 00:00 2023-11-17T05:18:59.134 INFO:teuthology.orchestra.run.smithi040.stdout:Running transaction check 2023-11-17T05:18:59.159 INFO:teuthology.orchestra.run.smithi040.stdout:Transaction check succeeded. 2023-11-17T05:18:59.159 INFO:teuthology.orchestra.run.smithi040.stdout:Running transaction test 2023-11-17T05:18:59.260 INFO:teuthology.orchestra.run.smithi040.stdout:Transaction test succeeded. 2023-11-17T05:18:59.263 INFO:teuthology.orchestra.run.smithi040.stdout:Running transaction 2023-11-17T05:18:59.454 INFO:teuthology.orchestra.run.smithi040.stdout: Preparing : 1/1 2023-11-17T05:18:59.755 INFO:teuthology.orchestra.run.smithi040.stdout: Installing : fuse-2.9.7-17.el8.x86_64 1/2 2023-11-17T05:18:59.972 INFO:teuthology.orchestra.run.smithi040.stdout: Installing : ceph-fuse-2:16.2.4-0.el8.x86_64 2/2 2023-11-17T05:19:00.004 INFO:teuthology.orchestra.run.smithi066.stdout:Last metadata expiration check: 0:03:47 ago on Fri 17 Nov 2023 05:15:13 AM UTC. 2023-11-17T05:19:00.429 INFO:teuthology.orchestra.run.smithi040.stdout: Running scriptlet: ceph-fuse-2:16.2.4-0.el8.x86_64 2/2 2023-11-17T05:19:00.429 INFO:teuthology.orchestra.run.smithi040.stdout: Verifying : ceph-fuse-2:16.2.4-0.el8.x86_64 1/2 2023-11-17T05:19:01.198 INFO:teuthology.orchestra.run.smithi040.stdout: Verifying : fuse-2.9.7-17.el8.x86_64 2/2 2023-11-17T05:19:01.198 INFO:teuthology.orchestra.run.smithi040.stdout: 2023-11-17T05:19:01.198 INFO:teuthology.orchestra.run.smithi040.stdout:Installed: 2023-11-17T05:19:01.198 INFO:teuthology.orchestra.run.smithi040.stdout: ceph-fuse-2:16.2.4-0.el8.x86_64 fuse-2.9.7-17.el8.x86_64 2023-11-17T05:19:01.198 INFO:teuthology.orchestra.run.smithi040.stdout: 2023-11-17T05:19:01.199 INFO:teuthology.orchestra.run.smithi040.stdout:Complete! 2023-11-17T05:19:01.434 DEBUG:teuthology.orchestra.run.smithi040:> sudo yum -y install librados-devel 2023-11-17T05:19:01.795 INFO:teuthology.orchestra.run.smithi066.stdout:Dependencies resolved. 2023-11-17T05:19:01.795 INFO:teuthology.orchestra.run.smithi066.stdout:================================================================================ 2023-11-17T05:19:01.795 INFO:teuthology.orchestra.run.smithi066.stdout: Package Architecture Version Repository Size 2023-11-17T05:19:01.796 INFO:teuthology.orchestra.run.smithi066.stdout:================================================================================ 2023-11-17T05:19:01.796 INFO:teuthology.orchestra.run.smithi066.stdout:Installing: 2023-11-17T05:19:01.796 INFO:teuthology.orchestra.run.smithi066.stdout: libcephfs-devel x86_64 2:16.2.4-0.el8 ceph 24 k 2023-11-17T05:19:01.796 INFO:teuthology.orchestra.run.smithi066.stdout: 2023-11-17T05:19:01.796 INFO:teuthology.orchestra.run.smithi066.stdout:Transaction Summary 2023-11-17T05:19:01.796 INFO:teuthology.orchestra.run.smithi066.stdout:================================================================================ 2023-11-17T05:19:01.797 INFO:teuthology.orchestra.run.smithi066.stdout:Install 1 Package 2023-11-17T05:19:01.797 INFO:teuthology.orchestra.run.smithi066.stdout: 2023-11-17T05:19:01.798 INFO:teuthology.orchestra.run.smithi066.stdout:Total download size: 24 k 2023-11-17T05:19:01.798 INFO:teuthology.orchestra.run.smithi066.stdout:Installed size: 94 k 2023-11-17T05:19:01.798 INFO:teuthology.orchestra.run.smithi066.stdout:Downloading Packages: 2023-11-17T05:19:01.841 INFO:teuthology.orchestra.run.smithi066.stdout:libcephfs-devel-16.2.4-0.el8.x86_64.rpm 543 kB/s | 24 kB 00:00 2023-11-17T05:19:01.842 INFO:teuthology.orchestra.run.smithi066.stdout:-------------------------------------------------------------------------------- 2023-11-17T05:19:01.842 INFO:teuthology.orchestra.run.smithi066.stdout:Total 528 kB/s | 24 kB 00:00 2023-11-17T05:19:01.842 INFO:teuthology.orchestra.run.smithi066.stdout:Running transaction check 2023-11-17T05:19:01.846 INFO:teuthology.orchestra.run.smithi066.stdout:Transaction check succeeded. 2023-11-17T05:19:01.846 INFO:teuthology.orchestra.run.smithi066.stdout:Running transaction test 2023-11-17T05:19:01.848 INFO:teuthology.orchestra.run.smithi066.stdout:Transaction test succeeded. 2023-11-17T05:19:01.852 INFO:teuthology.orchestra.run.smithi066.stdout:Running transaction 2023-11-17T05:19:01.906 INFO:teuthology.orchestra.run.smithi066.stdout: Preparing : 1/1 2023-11-17T05:19:02.093 INFO:teuthology.orchestra.run.smithi066.stdout: Installing : libcephfs-devel-2:16.2.4-0.el8.x86_64 1/1 2023-11-17T05:19:02.366 INFO:teuthology.orchestra.run.smithi066.stdout: Running scriptlet: libcephfs-devel-2:16.2.4-0.el8.x86_64 1/1 2023-11-17T05:19:02.437 INFO:teuthology.orchestra.run.smithi040.stdout:Last metadata expiration check: 0:03:44 ago on Fri 17 Nov 2023 05:15:18 AM UTC. 2023-11-17T05:19:03.050 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : libcephfs-devel-2:16.2.4-0.el8.x86_64 1/1 2023-11-17T05:19:03.051 INFO:teuthology.orchestra.run.smithi066.stdout: 2023-11-17T05:19:03.051 INFO:teuthology.orchestra.run.smithi066.stdout:Installed: 2023-11-17T05:19:03.051 INFO:teuthology.orchestra.run.smithi066.stdout: libcephfs-devel-2:16.2.4-0.el8.x86_64 2023-11-17T05:19:03.052 INFO:teuthology.orchestra.run.smithi066.stdout: 2023-11-17T05:19:03.052 INFO:teuthology.orchestra.run.smithi066.stdout:Complete! 2023-11-17T05:19:03.174 DEBUG:teuthology.orchestra.run.smithi066:> sudo yum -y install librados2 2023-11-17T05:19:04.077 INFO:teuthology.orchestra.run.smithi066.stdout:Last metadata expiration check: 0:03:51 ago on Fri 17 Nov 2023 05:15:13 AM UTC. 2023-11-17T05:19:04.297 INFO:teuthology.orchestra.run.smithi040.stdout:Dependencies resolved. 2023-11-17T05:19:04.298 INFO:teuthology.orchestra.run.smithi040.stdout:================================================================================ 2023-11-17T05:19:04.298 INFO:teuthology.orchestra.run.smithi040.stdout: Package Architecture Version Repository Size 2023-11-17T05:19:04.299 INFO:teuthology.orchestra.run.smithi040.stdout:================================================================================ 2023-11-17T05:19:04.299 INFO:teuthology.orchestra.run.smithi040.stdout:Installing: 2023-11-17T05:19:04.299 INFO:teuthology.orchestra.run.smithi040.stdout: librados-devel x86_64 2:16.2.4-0.el8 ceph 131 k 2023-11-17T05:19:04.299 INFO:teuthology.orchestra.run.smithi040.stdout: 2023-11-17T05:19:04.299 INFO:teuthology.orchestra.run.smithi040.stdout:Transaction Summary 2023-11-17T05:19:04.299 INFO:teuthology.orchestra.run.smithi040.stdout:================================================================================ 2023-11-17T05:19:04.299 INFO:teuthology.orchestra.run.smithi040.stdout:Install 1 Package 2023-11-17T05:19:04.299 INFO:teuthology.orchestra.run.smithi040.stdout: 2023-11-17T05:19:04.299 INFO:teuthology.orchestra.run.smithi040.stdout:Total download size: 131 k 2023-11-17T05:19:04.299 INFO:teuthology.orchestra.run.smithi040.stdout:Installed size: 439 k 2023-11-17T05:19:04.299 INFO:teuthology.orchestra.run.smithi040.stdout:Downloading Packages: 2023-11-17T05:19:04.761 INFO:teuthology.orchestra.run.smithi040.stdout:librados-devel-16.2.4-0.el8.x86_64.rpm 285 kB/s | 131 kB 00:00 2023-11-17T05:19:04.761 INFO:teuthology.orchestra.run.smithi040.stdout:-------------------------------------------------------------------------------- 2023-11-17T05:19:04.762 INFO:teuthology.orchestra.run.smithi040.stdout:Total 283 kB/s | 131 kB 00:00 2023-11-17T05:19:04.762 INFO:teuthology.orchestra.run.smithi040.stdout:Running transaction check 2023-11-17T05:19:04.769 INFO:teuthology.orchestra.run.smithi040.stdout:Transaction check succeeded. 2023-11-17T05:19:04.769 INFO:teuthology.orchestra.run.smithi040.stdout:Running transaction test 2023-11-17T05:19:04.830 INFO:teuthology.orchestra.run.smithi040.stdout:Transaction test succeeded. 2023-11-17T05:19:04.833 INFO:teuthology.orchestra.run.smithi040.stdout:Running transaction 2023-11-17T05:19:04.958 INFO:teuthology.orchestra.run.smithi040.stdout: Preparing : 1/1 2023-11-17T05:19:05.229 INFO:teuthology.orchestra.run.smithi040.stdout: Installing : librados-devel-2:16.2.4-0.el8.x86_64 1/1 2023-11-17T05:19:05.561 INFO:teuthology.orchestra.run.smithi040.stdout: Running scriptlet: librados-devel-2:16.2.4-0.el8.x86_64 1/1 2023-11-17T05:19:05.635 INFO:teuthology.orchestra.run.smithi066.stdout:Package librados2-2:16.2.4-0.el8.x86_64 is already installed. 2023-11-17T05:19:05.864 INFO:teuthology.orchestra.run.smithi066.stdout:Dependencies resolved. 2023-11-17T05:19:05.866 INFO:teuthology.orchestra.run.smithi066.stdout:Nothing to do. 2023-11-17T05:19:05.866 INFO:teuthology.orchestra.run.smithi066.stdout:Complete! 2023-11-17T05:19:05.972 DEBUG:teuthology.orchestra.run.smithi066:> sudo yum -y install librbd1 2023-11-17T05:19:06.300 INFO:teuthology.orchestra.run.smithi040.stdout: Verifying : librados-devel-2:16.2.4-0.el8.x86_64 1/1 2023-11-17T05:19:06.301 INFO:teuthology.orchestra.run.smithi040.stdout: 2023-11-17T05:19:06.301 INFO:teuthology.orchestra.run.smithi040.stdout:Installed: 2023-11-17T05:19:06.328 INFO:teuthology.orchestra.run.smithi040.stdout: librados-devel-2:16.2.4-0.el8.x86_64 2023-11-17T05:19:06.328 INFO:teuthology.orchestra.run.smithi040.stdout: 2023-11-17T05:19:06.328 INFO:teuthology.orchestra.run.smithi040.stdout:Complete! 2023-11-17T05:19:06.534 DEBUG:teuthology.orchestra.run.smithi040:> sudo yum -y install libcephfs2 2023-11-17T05:19:06.868 INFO:teuthology.orchestra.run.smithi066.stdout:Last metadata expiration check: 0:03:53 ago on Fri 17 Nov 2023 05:15:13 AM UTC. 2023-11-17T05:19:07.535 INFO:teuthology.orchestra.run.smithi040.stdout:Last metadata expiration check: 0:03:49 ago on Fri 17 Nov 2023 05:15:18 AM UTC. 2023-11-17T05:19:08.428 INFO:teuthology.orchestra.run.smithi066.stdout:Package librbd1-2:16.2.4-0.el8.x86_64 is already installed. 2023-11-17T05:19:08.657 INFO:teuthology.orchestra.run.smithi066.stdout:Dependencies resolved. 2023-11-17T05:19:08.659 INFO:teuthology.orchestra.run.smithi066.stdout:Nothing to do. 2023-11-17T05:19:08.659 INFO:teuthology.orchestra.run.smithi066.stdout:Complete! 2023-11-17T05:19:08.860 DEBUG:teuthology.orchestra.run.smithi066:> sudo yum -y install python3-rados 2023-11-17T05:19:09.250 INFO:teuthology.orchestra.run.smithi040.stdout:Package libcephfs2-2:16.2.4-0.el8.x86_64 is already installed. 2023-11-17T05:19:09.492 INFO:teuthology.orchestra.run.smithi040.stdout:Dependencies resolved. 2023-11-17T05:19:09.493 INFO:teuthology.orchestra.run.smithi040.stdout:Nothing to do. 2023-11-17T05:19:09.494 INFO:teuthology.orchestra.run.smithi040.stdout:Complete! 2023-11-17T05:19:09.619 DEBUG:teuthology.orchestra.run.smithi040:> sudo yum -y install libcephfs-devel 2023-11-17T05:19:09.762 INFO:teuthology.orchestra.run.smithi066.stdout:Last metadata expiration check: 0:03:56 ago on Fri 17 Nov 2023 05:15:13 AM UTC. 2023-11-17T05:19:10.625 INFO:teuthology.orchestra.run.smithi040.stdout:Last metadata expiration check: 0:03:52 ago on Fri 17 Nov 2023 05:15:18 AM UTC. 2023-11-17T05:19:11.299 INFO:teuthology.orchestra.run.smithi066.stdout:Package python3-rados-2:16.2.4-0.el8.x86_64 is already installed. 2023-11-17T05:19:11.529 INFO:teuthology.orchestra.run.smithi066.stdout:Dependencies resolved. 2023-11-17T05:19:11.530 INFO:teuthology.orchestra.run.smithi066.stdout:Nothing to do. 2023-11-17T05:19:11.530 INFO:teuthology.orchestra.run.smithi066.stdout:Complete! 2023-11-17T05:19:11.780 DEBUG:teuthology.orchestra.run.smithi066:> sudo yum -y install python3-rgw 2023-11-17T05:19:12.478 INFO:teuthology.orchestra.run.smithi040.stdout:Dependencies resolved. 2023-11-17T05:19:12.478 INFO:teuthology.orchestra.run.smithi040.stdout:================================================================================ 2023-11-17T05:19:12.479 INFO:teuthology.orchestra.run.smithi040.stdout: Package Architecture Version Repository Size 2023-11-17T05:19:12.479 INFO:teuthology.orchestra.run.smithi040.stdout:================================================================================ 2023-11-17T05:19:12.479 INFO:teuthology.orchestra.run.smithi040.stdout:Installing: 2023-11-17T05:19:12.479 INFO:teuthology.orchestra.run.smithi040.stdout: libcephfs-devel x86_64 2:16.2.4-0.el8 ceph 24 k 2023-11-17T05:19:12.479 INFO:teuthology.orchestra.run.smithi040.stdout: 2023-11-17T05:19:12.479 INFO:teuthology.orchestra.run.smithi040.stdout:Transaction Summary 2023-11-17T05:19:12.479 INFO:teuthology.orchestra.run.smithi040.stdout:================================================================================ 2023-11-17T05:19:12.479 INFO:teuthology.orchestra.run.smithi040.stdout:Install 1 Package 2023-11-17T05:19:12.479 INFO:teuthology.orchestra.run.smithi040.stdout: 2023-11-17T05:19:12.479 INFO:teuthology.orchestra.run.smithi040.stdout:Total download size: 24 k 2023-11-17T05:19:12.479 INFO:teuthology.orchestra.run.smithi040.stdout:Installed size: 94 k 2023-11-17T05:19:12.479 INFO:teuthology.orchestra.run.smithi040.stdout:Downloading Packages: 2023-11-17T05:19:12.522 INFO:teuthology.orchestra.run.smithi040.stdout:libcephfs-devel-16.2.4-0.el8.x86_64.rpm 556 kB/s | 24 kB 00:00 2023-11-17T05:19:12.523 INFO:teuthology.orchestra.run.smithi040.stdout:-------------------------------------------------------------------------------- 2023-11-17T05:19:12.523 INFO:teuthology.orchestra.run.smithi040.stdout:Total 539 kB/s | 24 kB 00:00 2023-11-17T05:19:12.523 INFO:teuthology.orchestra.run.smithi040.stdout:Running transaction check 2023-11-17T05:19:12.527 INFO:teuthology.orchestra.run.smithi040.stdout:Transaction check succeeded. 2023-11-17T05:19:12.527 INFO:teuthology.orchestra.run.smithi040.stdout:Running transaction test 2023-11-17T05:19:12.530 INFO:teuthology.orchestra.run.smithi040.stdout:Transaction test succeeded. 2023-11-17T05:19:12.534 INFO:teuthology.orchestra.run.smithi040.stdout:Running transaction 2023-11-17T05:19:12.583 INFO:teuthology.orchestra.run.smithi040.stdout: Preparing : 1/1 2023-11-17T05:19:12.677 INFO:teuthology.orchestra.run.smithi066.stdout:Last metadata expiration check: 0:03:59 ago on Fri 17 Nov 2023 05:15:13 AM UTC. 2023-11-17T05:19:12.802 INFO:teuthology.orchestra.run.smithi040.stdout: Installing : libcephfs-devel-2:16.2.4-0.el8.x86_64 1/1 2023-11-17T05:19:13.105 INFO:teuthology.orchestra.run.smithi040.stdout: Running scriptlet: libcephfs-devel-2:16.2.4-0.el8.x86_64 1/1 2023-11-17T05:19:13.800 INFO:teuthology.orchestra.run.smithi040.stdout: Verifying : libcephfs-devel-2:16.2.4-0.el8.x86_64 1/1 2023-11-17T05:19:13.832 INFO:teuthology.orchestra.run.smithi040.stdout: 2023-11-17T05:19:13.832 INFO:teuthology.orchestra.run.smithi040.stdout:Installed: 2023-11-17T05:19:13.832 INFO:teuthology.orchestra.run.smithi040.stdout: libcephfs-devel-2:16.2.4-0.el8.x86_64 2023-11-17T05:19:13.832 INFO:teuthology.orchestra.run.smithi040.stdout: 2023-11-17T05:19:13.832 INFO:teuthology.orchestra.run.smithi040.stdout:Complete! 2023-11-17T05:19:13.938 DEBUG:teuthology.orchestra.run.smithi040:> sudo yum -y install librados2 2023-11-17T05:19:14.256 INFO:teuthology.orchestra.run.smithi066.stdout:Package python3-rgw-2:16.2.4-0.el8.x86_64 is already installed. 2023-11-17T05:19:14.492 INFO:teuthology.orchestra.run.smithi066.stdout:Dependencies resolved. 2023-11-17T05:19:14.494 INFO:teuthology.orchestra.run.smithi066.stdout:Nothing to do. 2023-11-17T05:19:14.494 INFO:teuthology.orchestra.run.smithi066.stdout:Complete! 2023-11-17T05:19:14.598 DEBUG:teuthology.orchestra.run.smithi066:> sudo yum -y install python3-cephfs 2023-11-17T05:19:14.933 INFO:teuthology.orchestra.run.smithi040.stdout:Last metadata expiration check: 0:03:56 ago on Fri 17 Nov 2023 05:15:18 AM UTC. 2023-11-17T05:19:15.506 INFO:teuthology.orchestra.run.smithi066.stdout:Last metadata expiration check: 0:04:02 ago on Fri 17 Nov 2023 05:15:13 AM UTC. 2023-11-17T05:19:16.556 INFO:teuthology.orchestra.run.smithi040.stdout:Package librados2-2:16.2.4-0.el8.x86_64 is already installed. 2023-11-17T05:19:16.799 INFO:teuthology.orchestra.run.smithi040.stdout:Dependencies resolved. 2023-11-17T05:19:16.799 INFO:teuthology.orchestra.run.smithi040.stdout:Nothing to do. 2023-11-17T05:19:16.799 INFO:teuthology.orchestra.run.smithi040.stdout:Complete! 2023-11-17T05:19:16.927 DEBUG:teuthology.orchestra.run.smithi040:> sudo yum -y install librbd1 2023-11-17T05:19:17.068 INFO:teuthology.orchestra.run.smithi066.stdout:Package python3-cephfs-2:16.2.4-0.el8.x86_64 is already installed. 2023-11-17T05:19:17.302 INFO:teuthology.orchestra.run.smithi066.stdout:Dependencies resolved. 2023-11-17T05:19:17.303 INFO:teuthology.orchestra.run.smithi066.stdout:Nothing to do. 2023-11-17T05:19:17.303 INFO:teuthology.orchestra.run.smithi066.stdout:Complete! 2023-11-17T05:19:17.409 DEBUG:teuthology.orchestra.run.smithi066:> sudo yum -y install python3-rbd 2023-11-17T05:19:17.935 INFO:teuthology.orchestra.run.smithi040.stdout:Last metadata expiration check: 0:03:59 ago on Fri 17 Nov 2023 05:15:18 AM UTC. 2023-11-17T05:19:18.306 INFO:teuthology.orchestra.run.smithi066.stdout:Last metadata expiration check: 0:04:05 ago on Fri 17 Nov 2023 05:15:13 AM UTC. 2023-11-17T05:19:19.566 INFO:teuthology.orchestra.run.smithi040.stdout:Package librbd1-2:16.2.4-0.el8.x86_64 is already installed. 2023-11-17T05:19:19.808 INFO:teuthology.orchestra.run.smithi040.stdout:Dependencies resolved. 2023-11-17T05:19:19.810 INFO:teuthology.orchestra.run.smithi040.stdout:Nothing to do. 2023-11-17T05:19:19.810 INFO:teuthology.orchestra.run.smithi040.stdout:Complete! 2023-11-17T05:19:19.859 INFO:teuthology.orchestra.run.smithi066.stdout:Package python3-rbd-2:16.2.4-0.el8.x86_64 is already installed. 2023-11-17T05:19:19.933 DEBUG:teuthology.orchestra.run.smithi040:> sudo yum -y install python3-rados 2023-11-17T05:19:20.090 INFO:teuthology.orchestra.run.smithi066.stdout:Dependencies resolved. 2023-11-17T05:19:20.091 INFO:teuthology.orchestra.run.smithi066.stdout:Nothing to do. 2023-11-17T05:19:20.091 INFO:teuthology.orchestra.run.smithi066.stdout:Complete! 2023-11-17T05:19:20.206 DEBUG:teuthology.orchestra.run.smithi066:> sudo yum -y install rbd-fuse 2023-11-17T05:19:20.927 INFO:teuthology.orchestra.run.smithi040.stdout:Last metadata expiration check: 0:04:02 ago on Fri 17 Nov 2023 05:15:18 AM UTC. 2023-11-17T05:19:21.106 INFO:teuthology.orchestra.run.smithi066.stdout:Last metadata expiration check: 0:04:08 ago on Fri 17 Nov 2023 05:15:13 AM UTC. 2023-11-17T05:19:22.533 INFO:teuthology.orchestra.run.smithi040.stdout:Package python3-rados-2:16.2.4-0.el8.x86_64 is already installed. 2023-11-17T05:19:22.775 INFO:teuthology.orchestra.run.smithi040.stdout:Dependencies resolved. 2023-11-17T05:19:22.776 INFO:teuthology.orchestra.run.smithi040.stdout:Nothing to do. 2023-11-17T05:19:22.776 INFO:teuthology.orchestra.run.smithi040.stdout:Complete! 2023-11-17T05:19:22.882 INFO:teuthology.orchestra.run.smithi066.stdout:Dependencies resolved. 2023-11-17T05:19:22.882 INFO:teuthology.orchestra.run.smithi066.stdout:================================================================================ 2023-11-17T05:19:22.883 INFO:teuthology.orchestra.run.smithi066.stdout: Package Architecture Version Repository Size 2023-11-17T05:19:22.883 INFO:teuthology.orchestra.run.smithi066.stdout:================================================================================ 2023-11-17T05:19:22.883 INFO:teuthology.orchestra.run.smithi066.stdout:Installing: 2023-11-17T05:19:22.883 INFO:teuthology.orchestra.run.smithi066.stdout: rbd-fuse x86_64 2:16.2.4-0.el8 ceph 89 k 2023-11-17T05:19:22.883 INFO:teuthology.orchestra.run.smithi066.stdout: 2023-11-17T05:19:22.883 INFO:teuthology.orchestra.run.smithi066.stdout:Transaction Summary 2023-11-17T05:19:22.883 INFO:teuthology.orchestra.run.smithi066.stdout:================================================================================ 2023-11-17T05:19:22.883 INFO:teuthology.orchestra.run.smithi066.stdout:Install 1 Package 2023-11-17T05:19:22.883 INFO:teuthology.orchestra.run.smithi066.stdout: 2023-11-17T05:19:22.884 INFO:teuthology.orchestra.run.smithi066.stdout:Total download size: 89 k 2023-11-17T05:19:22.884 INFO:teuthology.orchestra.run.smithi066.stdout:Installed size: 298 k 2023-11-17T05:19:22.884 INFO:teuthology.orchestra.run.smithi066.stdout:Downloading Packages: 2023-11-17T05:19:22.901 DEBUG:teuthology.orchestra.run.smithi040:> sudo yum -y install python3-rgw 2023-11-17T05:19:22.957 INFO:teuthology.orchestra.run.smithi066.stdout:rbd-fuse-16.2.4-0.el8.x86_64.rpm 1.2 MB/s | 89 kB 00:00 2023-11-17T05:19:22.957 INFO:teuthology.orchestra.run.smithi066.stdout:-------------------------------------------------------------------------------- 2023-11-17T05:19:22.957 INFO:teuthology.orchestra.run.smithi066.stdout:Total 1.2 MB/s | 89 kB 00:00 2023-11-17T05:19:22.958 INFO:teuthology.orchestra.run.smithi066.stdout:Running transaction check 2023-11-17T05:19:22.968 INFO:teuthology.orchestra.run.smithi066.stdout:Transaction check succeeded. 2023-11-17T05:19:22.968 INFO:teuthology.orchestra.run.smithi066.stdout:Running transaction test 2023-11-17T05:19:23.023 INFO:teuthology.orchestra.run.smithi066.stdout:Transaction test succeeded. 2023-11-17T05:19:23.027 INFO:teuthology.orchestra.run.smithi066.stdout:Running transaction 2023-11-17T05:19:23.139 INFO:teuthology.orchestra.run.smithi066.stdout: Preparing : 1/1 2023-11-17T05:19:23.333 INFO:teuthology.orchestra.run.smithi066.stdout: Installing : rbd-fuse-2:16.2.4-0.el8.x86_64 1/1 2023-11-17T05:19:23.628 INFO:teuthology.orchestra.run.smithi066.stdout: Running scriptlet: rbd-fuse-2:16.2.4-0.el8.x86_64 1/1 2023-11-17T05:19:23.906 INFO:teuthology.orchestra.run.smithi040.stdout:Last metadata expiration check: 0:04:05 ago on Fri 17 Nov 2023 05:15:18 AM UTC. 2023-11-17T05:19:24.349 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : rbd-fuse-2:16.2.4-0.el8.x86_64 1/1 2023-11-17T05:19:24.349 INFO:teuthology.orchestra.run.smithi066.stdout: 2023-11-17T05:19:24.350 INFO:teuthology.orchestra.run.smithi066.stdout:Installed: 2023-11-17T05:19:24.350 INFO:teuthology.orchestra.run.smithi066.stdout: rbd-fuse-2:16.2.4-0.el8.x86_64 2023-11-17T05:19:24.351 INFO:teuthology.orchestra.run.smithi066.stdout: 2023-11-17T05:19:24.351 INFO:teuthology.orchestra.run.smithi066.stdout:Complete! 2023-11-17T05:19:24.644 DEBUG:teuthology.orchestra.run.smithi066:> sudo yum -y install rbd-mirror 2023-11-17T05:19:25.528 INFO:teuthology.orchestra.run.smithi040.stdout:Package python3-rgw-2:16.2.4-0.el8.x86_64 is already installed. 2023-11-17T05:19:25.548 INFO:teuthology.orchestra.run.smithi066.stdout:Last metadata expiration check: 0:04:12 ago on Fri 17 Nov 2023 05:15:13 AM UTC. 2023-11-17T05:19:25.768 INFO:teuthology.orchestra.run.smithi040.stdout:Dependencies resolved. 2023-11-17T05:19:25.770 INFO:teuthology.orchestra.run.smithi040.stdout:Nothing to do. 2023-11-17T05:19:25.770 INFO:teuthology.orchestra.run.smithi040.stdout:Complete! 2023-11-17T05:19:25.893 DEBUG:teuthology.orchestra.run.smithi040:> sudo yum -y install python3-cephfs 2023-11-17T05:19:26.885 INFO:teuthology.orchestra.run.smithi040.stdout:Last metadata expiration check: 0:04:08 ago on Fri 17 Nov 2023 05:15:18 AM UTC. 2023-11-17T05:19:27.328 INFO:teuthology.orchestra.run.smithi066.stdout:Dependencies resolved. 2023-11-17T05:19:27.329 INFO:teuthology.orchestra.run.smithi066.stdout:================================================================================ 2023-11-17T05:19:27.329 INFO:teuthology.orchestra.run.smithi066.stdout: Package Architecture Version Repository Size 2023-11-17T05:19:27.329 INFO:teuthology.orchestra.run.smithi066.stdout:================================================================================ 2023-11-17T05:19:27.329 INFO:teuthology.orchestra.run.smithi066.stdout:Installing: 2023-11-17T05:19:27.329 INFO:teuthology.orchestra.run.smithi066.stdout: rbd-mirror x86_64 2:16.2.4-0.el8 ceph 3.7 M 2023-11-17T05:19:27.329 INFO:teuthology.orchestra.run.smithi066.stdout: 2023-11-17T05:19:27.330 INFO:teuthology.orchestra.run.smithi066.stdout:Transaction Summary 2023-11-17T05:19:27.330 INFO:teuthology.orchestra.run.smithi066.stdout:================================================================================ 2023-11-17T05:19:27.330 INFO:teuthology.orchestra.run.smithi066.stdout:Install 1 Package 2023-11-17T05:19:27.330 INFO:teuthology.orchestra.run.smithi066.stdout: 2023-11-17T05:19:27.330 INFO:teuthology.orchestra.run.smithi066.stdout:Total download size: 3.7 M 2023-11-17T05:19:27.330 INFO:teuthology.orchestra.run.smithi066.stdout:Installed size: 18 M 2023-11-17T05:19:27.331 INFO:teuthology.orchestra.run.smithi066.stdout:Downloading Packages: 2023-11-17T05:19:27.467 INFO:teuthology.orchestra.run.smithi066.stdout:rbd-mirror-16.2.4-0.el8.x86_64.rpm 27 MB/s | 3.7 MB 00:00 2023-11-17T05:19:27.468 INFO:teuthology.orchestra.run.smithi066.stdout:-------------------------------------------------------------------------------- 2023-11-17T05:19:27.468 INFO:teuthology.orchestra.run.smithi066.stdout:Total 27 MB/s | 3.7 MB 00:00 2023-11-17T05:19:27.469 INFO:teuthology.orchestra.run.smithi066.stdout:Running transaction check 2023-11-17T05:19:27.482 INFO:teuthology.orchestra.run.smithi066.stdout:Transaction check succeeded. 2023-11-17T05:19:27.482 INFO:teuthology.orchestra.run.smithi066.stdout:Running transaction test 2023-11-17T05:19:27.549 INFO:teuthology.orchestra.run.smithi066.stdout:Transaction test succeeded. 2023-11-17T05:19:27.552 INFO:teuthology.orchestra.run.smithi066.stdout:Running transaction 2023-11-17T05:19:28.043 INFO:teuthology.orchestra.run.smithi066.stdout: Preparing : 1/1 2023-11-17T05:19:28.257 INFO:teuthology.orchestra.run.smithi066.stdout: Installing : rbd-mirror-2:16.2.4-0.el8.x86_64 1/1 2023-11-17T05:19:28.490 INFO:teuthology.orchestra.run.smithi040.stdout:Package python3-cephfs-2:16.2.4-0.el8.x86_64 is already installed. 2023-11-17T05:19:28.703 INFO:teuthology.orchestra.run.smithi066.stdout: Running scriptlet: rbd-mirror-2:16.2.4-0.el8.x86_64 1/1 2023-11-17T05:19:28.730 INFO:teuthology.orchestra.run.smithi040.stdout:Dependencies resolved. 2023-11-17T05:19:28.732 INFO:teuthology.orchestra.run.smithi040.stdout:Nothing to do. 2023-11-17T05:19:28.732 INFO:teuthology.orchestra.run.smithi040.stdout:Complete! 2023-11-17T05:19:28.853 DEBUG:teuthology.orchestra.run.smithi040:> sudo yum -y install python3-rbd 2023-11-17T05:19:29.388 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : rbd-mirror-2:16.2.4-0.el8.x86_64 1/1 2023-11-17T05:19:29.389 INFO:teuthology.orchestra.run.smithi066.stdout: 2023-11-17T05:19:29.389 INFO:teuthology.orchestra.run.smithi066.stdout:Installed: 2023-11-17T05:19:29.390 INFO:teuthology.orchestra.run.smithi066.stdout: rbd-mirror-2:16.2.4-0.el8.x86_64 2023-11-17T05:19:29.390 INFO:teuthology.orchestra.run.smithi066.stdout: 2023-11-17T05:19:29.390 INFO:teuthology.orchestra.run.smithi066.stdout:Complete! 2023-11-17T05:19:29.596 DEBUG:teuthology.orchestra.run.smithi066:> sudo yum -y install rbd-nbd 2023-11-17T05:19:29.860 INFO:teuthology.orchestra.run.smithi040.stdout:Last metadata expiration check: 0:04:11 ago on Fri 17 Nov 2023 05:15:18 AM UTC. 2023-11-17T05:19:30.505 INFO:teuthology.orchestra.run.smithi066.stdout:Last metadata expiration check: 0:04:17 ago on Fri 17 Nov 2023 05:15:13 AM UTC. 2023-11-17T05:19:31.481 INFO:teuthology.orchestra.run.smithi040.stdout:Package python3-rbd-2:16.2.4-0.el8.x86_64 is already installed. 2023-11-17T05:19:31.721 INFO:teuthology.orchestra.run.smithi040.stdout:Dependencies resolved. 2023-11-17T05:19:31.722 INFO:teuthology.orchestra.run.smithi040.stdout:Nothing to do. 2023-11-17T05:19:31.722 INFO:teuthology.orchestra.run.smithi040.stdout:Complete! 2023-11-17T05:19:31.835 DEBUG:teuthology.orchestra.run.smithi040:> sudo yum -y install rbd-fuse 2023-11-17T05:19:32.291 INFO:teuthology.orchestra.run.smithi066.stdout:Dependencies resolved. 2023-11-17T05:19:32.292 INFO:teuthology.orchestra.run.smithi066.stdout:================================================================================ 2023-11-17T05:19:32.292 INFO:teuthology.orchestra.run.smithi066.stdout: Package Architecture Version Repository Size 2023-11-17T05:19:32.292 INFO:teuthology.orchestra.run.smithi066.stdout:================================================================================ 2023-11-17T05:19:32.292 INFO:teuthology.orchestra.run.smithi066.stdout:Installing: 2023-11-17T05:19:32.292 INFO:teuthology.orchestra.run.smithi066.stdout: rbd-nbd x86_64 2:16.2.4-0.el8 ceph 173 k 2023-11-17T05:19:32.292 INFO:teuthology.orchestra.run.smithi066.stdout: 2023-11-17T05:19:32.292 INFO:teuthology.orchestra.run.smithi066.stdout:Transaction Summary 2023-11-17T05:19:32.293 INFO:teuthology.orchestra.run.smithi066.stdout:================================================================================ 2023-11-17T05:19:32.293 INFO:teuthology.orchestra.run.smithi066.stdout:Install 1 Package 2023-11-17T05:19:32.293 INFO:teuthology.orchestra.run.smithi066.stdout: 2023-11-17T05:19:32.293 INFO:teuthology.orchestra.run.smithi066.stdout:Total download size: 173 k 2023-11-17T05:19:32.294 INFO:teuthology.orchestra.run.smithi066.stdout:Installed size: 649 k 2023-11-17T05:19:32.294 INFO:teuthology.orchestra.run.smithi066.stdout:Downloading Packages: 2023-11-17T05:19:32.353 INFO:teuthology.orchestra.run.smithi066.stdout:rbd-nbd-16.2.4-0.el8.x86_64.rpm 2.8 MB/s | 173 kB 00:00 2023-11-17T05:19:32.354 INFO:teuthology.orchestra.run.smithi066.stdout:-------------------------------------------------------------------------------- 2023-11-17T05:19:32.354 INFO:teuthology.orchestra.run.smithi066.stdout:Total 2.8 MB/s | 173 kB 00:00 2023-11-17T05:19:32.354 INFO:teuthology.orchestra.run.smithi066.stdout:Running transaction check 2023-11-17T05:19:32.366 INFO:teuthology.orchestra.run.smithi066.stdout:Transaction check succeeded. 2023-11-17T05:19:32.366 INFO:teuthology.orchestra.run.smithi066.stdout:Running transaction test 2023-11-17T05:19:32.421 INFO:teuthology.orchestra.run.smithi066.stdout:Transaction test succeeded. 2023-11-17T05:19:32.425 INFO:teuthology.orchestra.run.smithi066.stdout:Running transaction 2023-11-17T05:19:32.545 INFO:teuthology.orchestra.run.smithi066.stdout: Preparing : 1/1 2023-11-17T05:19:32.765 INFO:teuthology.orchestra.run.smithi066.stdout: Installing : rbd-nbd-2:16.2.4-0.el8.x86_64 1/1 2023-11-17T05:19:32.832 INFO:teuthology.orchestra.run.smithi040.stdout:Last metadata expiration check: 0:04:14 ago on Fri 17 Nov 2023 05:15:18 AM UTC. 2023-11-17T05:19:33.100 INFO:teuthology.orchestra.run.smithi066.stdout: Running scriptlet: rbd-nbd-2:16.2.4-0.el8.x86_64 1/1 2023-11-17T05:19:33.802 INFO:teuthology.orchestra.run.smithi066.stdout: Verifying : rbd-nbd-2:16.2.4-0.el8.x86_64 1/1 2023-11-17T05:19:33.838 INFO:teuthology.orchestra.run.smithi066.stdout: 2023-11-17T05:19:33.838 INFO:teuthology.orchestra.run.smithi066.stdout:Installed: 2023-11-17T05:19:33.838 INFO:teuthology.orchestra.run.smithi066.stdout: rbd-nbd-2:16.2.4-0.el8.x86_64 2023-11-17T05:19:33.838 INFO:teuthology.orchestra.run.smithi066.stdout: 2023-11-17T05:19:33.838 INFO:teuthology.orchestra.run.smithi066.stdout:Complete! 2023-11-17T05:19:34.005 DEBUG:teuthology.parallel:result is None 2023-11-17T05:19:34.698 INFO:teuthology.orchestra.run.smithi040.stdout:Dependencies resolved. 2023-11-17T05:19:34.699 INFO:teuthology.orchestra.run.smithi040.stdout:================================================================================ 2023-11-17T05:19:34.699 INFO:teuthology.orchestra.run.smithi040.stdout: Package Architecture Version Repository Size 2023-11-17T05:19:34.699 INFO:teuthology.orchestra.run.smithi040.stdout:================================================================================ 2023-11-17T05:19:34.699 INFO:teuthology.orchestra.run.smithi040.stdout:Installing: 2023-11-17T05:19:34.699 INFO:teuthology.orchestra.run.smithi040.stdout: rbd-fuse x86_64 2:16.2.4-0.el8 ceph 89 k 2023-11-17T05:19:34.699 INFO:teuthology.orchestra.run.smithi040.stdout: 2023-11-17T05:19:34.699 INFO:teuthology.orchestra.run.smithi040.stdout:Transaction Summary 2023-11-17T05:19:34.699 INFO:teuthology.orchestra.run.smithi040.stdout:================================================================================ 2023-11-17T05:19:34.699 INFO:teuthology.orchestra.run.smithi040.stdout:Install 1 Package 2023-11-17T05:19:34.699 INFO:teuthology.orchestra.run.smithi040.stdout: 2023-11-17T05:19:34.700 INFO:teuthology.orchestra.run.smithi040.stdout:Total download size: 89 k 2023-11-17T05:19:34.700 INFO:teuthology.orchestra.run.smithi040.stdout:Installed size: 298 k 2023-11-17T05:19:34.700 INFO:teuthology.orchestra.run.smithi040.stdout:Downloading Packages: 2023-11-17T05:19:34.746 INFO:teuthology.orchestra.run.smithi040.stdout:rbd-fuse-16.2.4-0.el8.x86_64.rpm 1.9 MB/s | 89 kB 00:00 2023-11-17T05:19:34.746 INFO:teuthology.orchestra.run.smithi040.stdout:-------------------------------------------------------------------------------- 2023-11-17T05:19:34.747 INFO:teuthology.orchestra.run.smithi040.stdout:Total 1.8 MB/s | 89 kB 00:00 2023-11-17T05:19:34.747 INFO:teuthology.orchestra.run.smithi040.stdout:Running transaction check 2023-11-17T05:19:34.758 INFO:teuthology.orchestra.run.smithi040.stdout:Transaction check succeeded. 2023-11-17T05:19:34.758 INFO:teuthology.orchestra.run.smithi040.stdout:Running transaction test 2023-11-17T05:19:34.814 INFO:teuthology.orchestra.run.smithi040.stdout:Transaction test succeeded. 2023-11-17T05:19:34.818 INFO:teuthology.orchestra.run.smithi040.stdout:Running transaction 2023-11-17T05:19:34.939 INFO:teuthology.orchestra.run.smithi040.stdout: Preparing : 1/1 2023-11-17T05:19:35.174 INFO:teuthology.orchestra.run.smithi040.stdout: Installing : rbd-fuse-2:16.2.4-0.el8.x86_64 1/1 2023-11-17T05:19:35.511 INFO:teuthology.orchestra.run.smithi040.stdout: Running scriptlet: rbd-fuse-2:16.2.4-0.el8.x86_64 1/1 2023-11-17T05:19:36.271 INFO:teuthology.orchestra.run.smithi040.stdout: Verifying : rbd-fuse-2:16.2.4-0.el8.x86_64 1/1 2023-11-17T05:19:36.272 INFO:teuthology.orchestra.run.smithi040.stdout: 2023-11-17T05:19:36.272 INFO:teuthology.orchestra.run.smithi040.stdout:Installed: 2023-11-17T05:19:36.272 INFO:teuthology.orchestra.run.smithi040.stdout: rbd-fuse-2:16.2.4-0.el8.x86_64 2023-11-17T05:19:36.272 INFO:teuthology.orchestra.run.smithi040.stdout: 2023-11-17T05:19:36.272 INFO:teuthology.orchestra.run.smithi040.stdout:Complete! 2023-11-17T05:19:36.535 DEBUG:teuthology.orchestra.run.smithi040:> sudo yum -y install rbd-mirror 2023-11-17T05:19:37.533 INFO:teuthology.orchestra.run.smithi040.stdout:Last metadata expiration check: 0:04:19 ago on Fri 17 Nov 2023 05:15:18 AM UTC. 2023-11-17T05:19:39.379 INFO:teuthology.orchestra.run.smithi040.stdout:Dependencies resolved. 2023-11-17T05:19:39.380 INFO:teuthology.orchestra.run.smithi040.stdout:================================================================================ 2023-11-17T05:19:39.380 INFO:teuthology.orchestra.run.smithi040.stdout: Package Architecture Version Repository Size 2023-11-17T05:19:39.380 INFO:teuthology.orchestra.run.smithi040.stdout:================================================================================ 2023-11-17T05:19:39.380 INFO:teuthology.orchestra.run.smithi040.stdout:Installing: 2023-11-17T05:19:39.380 INFO:teuthology.orchestra.run.smithi040.stdout: rbd-mirror x86_64 2:16.2.4-0.el8 ceph 3.7 M 2023-11-17T05:19:39.380 INFO:teuthology.orchestra.run.smithi040.stdout: 2023-11-17T05:19:39.380 INFO:teuthology.orchestra.run.smithi040.stdout:Transaction Summary 2023-11-17T05:19:39.380 INFO:teuthology.orchestra.run.smithi040.stdout:================================================================================ 2023-11-17T05:19:39.380 INFO:teuthology.orchestra.run.smithi040.stdout:Install 1 Package 2023-11-17T05:19:39.380 INFO:teuthology.orchestra.run.smithi040.stdout: 2023-11-17T05:19:39.381 INFO:teuthology.orchestra.run.smithi040.stdout:Total download size: 3.7 M 2023-11-17T05:19:39.381 INFO:teuthology.orchestra.run.smithi040.stdout:Installed size: 18 M 2023-11-17T05:19:39.381 INFO:teuthology.orchestra.run.smithi040.stdout:Downloading Packages: 2023-11-17T05:19:39.465 INFO:teuthology.orchestra.run.smithi040.stdout:rbd-mirror-16.2.4-0.el8.x86_64.rpm 44 MB/s | 3.7 MB 00:00 2023-11-17T05:19:39.465 INFO:teuthology.orchestra.run.smithi040.stdout:-------------------------------------------------------------------------------- 2023-11-17T05:19:39.466 INFO:teuthology.orchestra.run.smithi040.stdout:Total 44 MB/s | 3.7 MB 00:00 2023-11-17T05:19:39.467 INFO:teuthology.orchestra.run.smithi040.stdout:Running transaction check 2023-11-17T05:19:39.480 INFO:teuthology.orchestra.run.smithi040.stdout:Transaction check succeeded. 2023-11-17T05:19:39.480 INFO:teuthology.orchestra.run.smithi040.stdout:Running transaction test 2023-11-17T05:19:39.550 INFO:teuthology.orchestra.run.smithi040.stdout:Transaction test succeeded. 2023-11-17T05:19:39.554 INFO:teuthology.orchestra.run.smithi040.stdout:Running transaction 2023-11-17T05:19:40.162 INFO:teuthology.orchestra.run.smithi040.stdout: Preparing : 1/1 2023-11-17T05:19:40.382 INFO:teuthology.orchestra.run.smithi040.stdout: Installing : rbd-mirror-2:16.2.4-0.el8.x86_64 1/1 2023-11-17T05:19:40.855 INFO:teuthology.orchestra.run.smithi040.stdout: Running scriptlet: rbd-mirror-2:16.2.4-0.el8.x86_64 1/1 2023-11-17T05:19:41.598 INFO:teuthology.orchestra.run.smithi040.stdout: Verifying : rbd-mirror-2:16.2.4-0.el8.x86_64 1/1 2023-11-17T05:19:41.599 INFO:teuthology.orchestra.run.smithi040.stdout: 2023-11-17T05:19:41.599 INFO:teuthology.orchestra.run.smithi040.stdout:Installed: 2023-11-17T05:19:41.599 INFO:teuthology.orchestra.run.smithi040.stdout: rbd-mirror-2:16.2.4-0.el8.x86_64 2023-11-17T05:19:41.599 INFO:teuthology.orchestra.run.smithi040.stdout: 2023-11-17T05:19:41.599 INFO:teuthology.orchestra.run.smithi040.stdout:Complete! 2023-11-17T05:19:41.818 DEBUG:teuthology.orchestra.run.smithi040:> sudo yum -y install rbd-nbd 2023-11-17T05:19:42.816 INFO:teuthology.orchestra.run.smithi040.stdout:Last metadata expiration check: 0:04:24 ago on Fri 17 Nov 2023 05:15:18 AM UTC. 2023-11-17T05:19:44.755 INFO:teuthology.orchestra.run.smithi040.stdout:Dependencies resolved. 2023-11-17T05:19:44.755 INFO:teuthology.orchestra.run.smithi040.stdout:================================================================================ 2023-11-17T05:19:44.756 INFO:teuthology.orchestra.run.smithi040.stdout: Package Architecture Version Repository Size 2023-11-17T05:19:44.756 INFO:teuthology.orchestra.run.smithi040.stdout:================================================================================ 2023-11-17T05:19:44.756 INFO:teuthology.orchestra.run.smithi040.stdout:Installing: 2023-11-17T05:19:44.756 INFO:teuthology.orchestra.run.smithi040.stdout: rbd-nbd x86_64 2:16.2.4-0.el8 ceph 173 k 2023-11-17T05:19:44.756 INFO:teuthology.orchestra.run.smithi040.stdout: 2023-11-17T05:19:44.756 INFO:teuthology.orchestra.run.smithi040.stdout:Transaction Summary 2023-11-17T05:19:44.756 INFO:teuthology.orchestra.run.smithi040.stdout:================================================================================ 2023-11-17T05:19:44.756 INFO:teuthology.orchestra.run.smithi040.stdout:Install 1 Package 2023-11-17T05:19:44.756 INFO:teuthology.orchestra.run.smithi040.stdout: 2023-11-17T05:19:44.757 INFO:teuthology.orchestra.run.smithi040.stdout:Total download size: 173 k 2023-11-17T05:19:44.757 INFO:teuthology.orchestra.run.smithi040.stdout:Installed size: 649 k 2023-11-17T05:19:44.757 INFO:teuthology.orchestra.run.smithi040.stdout:Downloading Packages: 2023-11-17T05:19:44.810 INFO:teuthology.orchestra.run.smithi040.stdout:rbd-nbd-16.2.4-0.el8.x86_64.rpm 3.3 MB/s | 173 kB 00:00 2023-11-17T05:19:44.810 INFO:teuthology.orchestra.run.smithi040.stdout:-------------------------------------------------------------------------------- 2023-11-17T05:19:44.810 INFO:teuthology.orchestra.run.smithi040.stdout:Total 3.2 MB/s | 173 kB 00:00 2023-11-17T05:19:44.811 INFO:teuthology.orchestra.run.smithi040.stdout:Running transaction check 2023-11-17T05:19:44.823 INFO:teuthology.orchestra.run.smithi040.stdout:Transaction check succeeded. 2023-11-17T05:19:44.823 INFO:teuthology.orchestra.run.smithi040.stdout:Running transaction test 2023-11-17T05:19:44.881 INFO:teuthology.orchestra.run.smithi040.stdout:Transaction test succeeded. 2023-11-17T05:19:44.885 INFO:teuthology.orchestra.run.smithi040.stdout:Running transaction 2023-11-17T05:19:45.016 INFO:teuthology.orchestra.run.smithi040.stdout: Preparing : 1/1 2023-11-17T05:19:45.223 INFO:teuthology.orchestra.run.smithi040.stdout: Installing : rbd-nbd-2:16.2.4-0.el8.x86_64 1/1 2023-11-17T05:19:45.602 INFO:teuthology.orchestra.run.smithi040.stdout: Running scriptlet: rbd-nbd-2:16.2.4-0.el8.x86_64 1/1 2023-11-17T05:19:46.404 INFO:teuthology.orchestra.run.smithi040.stdout: Verifying : rbd-nbd-2:16.2.4-0.el8.x86_64 1/1 2023-11-17T05:19:46.404 INFO:teuthology.orchestra.run.smithi040.stdout: 2023-11-17T05:19:46.404 INFO:teuthology.orchestra.run.smithi040.stdout:Installed: 2023-11-17T05:19:46.405 INFO:teuthology.orchestra.run.smithi040.stdout: rbd-nbd-2:16.2.4-0.el8.x86_64 2023-11-17T05:19:46.405 INFO:teuthology.orchestra.run.smithi040.stdout: 2023-11-17T05:19:46.405 INFO:teuthology.orchestra.run.smithi040.stdout:Complete! 2023-11-17T05:19:46.621 DEBUG:teuthology.parallel:result is None 2023-11-17T05:19:46.621 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using tag 2023-11-17T05:19:46.621 INFO:teuthology.packaging:ref: None 2023-11-17T05:19:46.621 INFO:teuthology.packaging:tag: v16.2.4 2023-11-17T05:19:46.621 INFO:teuthology.packaging:branch: None 2023-11-17T05:19:46.621 INFO:teuthology.packaging:sha1: ed3c8e926b9de93026c8a7c1d86be5d6821b52ac 2023-11-17T05:19:46.834 DEBUG:teuthology.repo_utils:git ls-remote https://github.com/ceph/ceph-ci v16.2.4^{} -> 3cbe25cde3cfa028984618ad32de9edc4c1eaed0 2023-11-17T05:19:46.941 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F8%2Fx86_64&sha1=3cbe25cde3cfa028984618ad32de9edc4c1eaed0 2023-11-17T05:19:47.090 DEBUG:teuthology.orchestra.run.smithi040:> rpm -q ceph --qf '%{VERSION}-%{RELEASE}' 2023-11-17T05:19:47.119 INFO:teuthology.orchestra.run.smithi040.stdout:16.2.4-0.el8 2023-11-17T05:19:47.119 INFO:teuthology.packaging:The installed version of ceph is 16.2.4-0.el8 2023-11-17T05:19:47.120 INFO:teuthology.task.install:The correct ceph version 16.2.4-0 is installed. 2023-11-17T05:19:47.120 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using tag 2023-11-17T05:19:47.120 INFO:teuthology.packaging:ref: None 2023-11-17T05:19:47.120 INFO:teuthology.packaging:tag: v16.2.4 2023-11-17T05:19:47.120 INFO:teuthology.packaging:branch: None 2023-11-17T05:19:47.120 INFO:teuthology.packaging:sha1: ed3c8e926b9de93026c8a7c1d86be5d6821b52ac 2023-11-17T05:19:47.308 DEBUG:teuthology.repo_utils:git ls-remote https://github.com/ceph/ceph-ci v16.2.4^{} -> 3cbe25cde3cfa028984618ad32de9edc4c1eaed0 2023-11-17T05:19:47.309 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F8%2Fx86_64&sha1=3cbe25cde3cfa028984618ad32de9edc4c1eaed0 2023-11-17T05:19:47.529 DEBUG:teuthology.orchestra.run.smithi066:> rpm -q ceph --qf '%{VERSION}-%{RELEASE}' 2023-11-17T05:19:47.559 INFO:teuthology.orchestra.run.smithi066.stdout:16.2.4-0.el8 2023-11-17T05:19:47.559 INFO:teuthology.packaging:The installed version of ceph is 16.2.4-0.el8 2023-11-17T05:19:47.559 INFO:teuthology.task.install:The correct ceph version 16.2.4-0 is installed. 2023-11-17T05:19:47.561 INFO:teuthology.task.install.util:Shipping valgrind.supp... 2023-11-17T05:19:47.561 DEBUG:teuthology.orchestra.run.smithi040:> set -ex 2023-11-17T05:19:47.562 DEBUG:teuthology.orchestra.run.smithi040:> sudo dd of=/home/ubuntu/cephtest/valgrind.supp 2023-11-17T05:19:47.598 DEBUG:teuthology.orchestra.run.smithi066:> set -ex 2023-11-17T05:19:47.598 DEBUG:teuthology.orchestra.run.smithi066:> sudo dd of=/home/ubuntu/cephtest/valgrind.supp 2023-11-17T05:19:47.632 INFO:teuthology.task.install.util:Shipping 'daemon-helper'... 2023-11-17T05:19:47.632 DEBUG:teuthology.orchestra.run.smithi040:> set -ex 2023-11-17T05:19:47.632 DEBUG:teuthology.orchestra.run.smithi040:> sudo dd of=/usr/bin/daemon-helper 2023-11-17T05:19:47.674 DEBUG:teuthology.orchestra.run.smithi040:> sudo chmod a=rx -- /usr/bin/daemon-helper 2023-11-17T05:19:47.748 DEBUG:teuthology.orchestra.run.smithi066:> set -ex 2023-11-17T05:19:47.748 DEBUG:teuthology.orchestra.run.smithi066:> sudo dd of=/usr/bin/daemon-helper 2023-11-17T05:19:47.785 DEBUG:teuthology.orchestra.run.smithi066:> sudo chmod a=rx -- /usr/bin/daemon-helper 2023-11-17T05:19:47.856 INFO:teuthology.task.install.util:Shipping 'adjust-ulimits'... 2023-11-17T05:19:47.856 DEBUG:teuthology.orchestra.run.smithi040:> set -ex 2023-11-17T05:19:47.856 DEBUG:teuthology.orchestra.run.smithi040:> sudo dd of=/usr/bin/adjust-ulimits 2023-11-17T05:19:47.890 DEBUG:teuthology.orchestra.run.smithi040:> sudo chmod a=rx -- /usr/bin/adjust-ulimits 2023-11-17T05:19:47.964 DEBUG:teuthology.orchestra.run.smithi066:> set -ex 2023-11-17T05:19:47.964 DEBUG:teuthology.orchestra.run.smithi066:> sudo dd of=/usr/bin/adjust-ulimits 2023-11-17T05:19:47.995 DEBUG:teuthology.orchestra.run.smithi066:> sudo chmod a=rx -- /usr/bin/adjust-ulimits 2023-11-17T05:19:48.068 INFO:teuthology.task.install.util:Shipping 'stdin-killer'... 2023-11-17T05:19:48.068 DEBUG:teuthology.orchestra.run.smithi040:> set -ex 2023-11-17T05:19:48.068 DEBUG:teuthology.orchestra.run.smithi040:> sudo dd of=/usr/bin/stdin-killer 2023-11-17T05:19:48.108 DEBUG:teuthology.orchestra.run.smithi040:> sudo chmod a=rx -- /usr/bin/stdin-killer 2023-11-17T05:19:48.184 DEBUG:teuthology.orchestra.run.smithi066:> set -ex 2023-11-17T05:19:48.184 DEBUG:teuthology.orchestra.run.smithi066:> sudo dd of=/usr/bin/stdin-killer 2023-11-17T05:19:48.215 DEBUG:teuthology.orchestra.run.smithi066:> sudo chmod a=rx -- /usr/bin/stdin-killer 2023-11-17T05:19:48.286 INFO:teuthology.run_tasks:Running task print... 2023-11-17T05:19:48.298 INFO:teuthology.task.print:**** done install task... 2023-11-17T05:19:48.298 INFO:teuthology.run_tasks:Running task cephadm... 2023-11-17T05:19:48.407 INFO:tasks.cephadm:Config: {'avoid_pacific_features': True, 'cephadm_branch': 'v16.2.4', 'cephadm_git_url': 'https://github.com/ceph/ceph', 'image': 'docker.io/ceph/ceph:v16.2.4', 'roleless': True, 'conf': {'client': {'client mount timeout': 600, 'debug client': 20, 'debug ms': 1, 'rados mon op timeout': 900, 'rados osd op timeout': 900}, 'global': {'mon pg warn min per osd': 0}, 'mds': {'debug mds': 20, 'debug ms': 1, 'mds debug frag': True, 'mds debug scatterstat': True, 'mds op complaint time': 180, 'mds verify scatter': True, 'osd op complaint time': 180, 'rados mon op timeout': 900, 'rados osd op timeout': 900}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20, 'mon op complaint time': 120}, 'osd': {'bdev async discard': True, 'bdev enable discard': True, 'bluestore allocator': 'bitmap', 'bluestore block size': 96636764160, 'bluestore fsck on mount': True, 'debug bluefs': '1/20', 'debug bluestore': '1/20', 'debug ms': 1, 'debug osd': 20, 'debug rocksdb': '4/10', 'mon osd backfillfull_ratio': 0.85, 'mon osd full ratio': 0.9, 'mon osd nearfull ratio': 0.8, 'osd failsafe full ratio': 0.95, 'osd objectstore': 'bluestore', 'osd op complaint time': 180}}, 'flavor': 'default', 'fs': 'xfs', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)', 'overall HEALTH_', '\\(FS_DEGRADED\\)', '\\(MDS_FAILED\\)', '\\(MDS_DEGRADED\\)', '\\(FS_WITH_FAILED_MDS\\)', '\\(MDS_DAMAGE\\)', '\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)', '\\(FS_INLINE_DATA_DEPRECATED\\)', 'overall HEALTH_', '\\(OSD_DOWN\\)', '\\(OSD_', 'but it is still running', 'is not responding'], 'sha1': 'ed3c8e926b9de93026c8a7c1d86be5d6821b52ac'} 2023-11-17T05:19:48.407 INFO:tasks.cephadm:Cluster image is docker.io/ceph/ceph:v16.2.4 2023-11-17T05:19:48.408 INFO:tasks.cephadm:Cluster fsid is ed6470e2-8508-11ee-8dbe-212e2dc638e7 2023-11-17T05:19:48.408 INFO:tasks.cephadm:Choosing monitor IPs and ports... 2023-11-17T05:19:48.408 INFO:tasks.cephadm:No mon roles; fabricating mons 2023-11-17T05:19:48.408 INFO:tasks.cephadm:Monitor IPs: {'mon.smithi040': '172.21.15.40', 'mon.smithi066': '172.21.15.66'} 2023-11-17T05:19:48.408 INFO:tasks.cephadm:Normalizing hostnames... 2023-11-17T05:19:48.408 DEBUG:teuthology.orchestra.run.smithi040:> sudo hostname $(hostname -s) 2023-11-17T05:19:48.450 DEBUG:teuthology.orchestra.run.smithi066:> sudo hostname $(hostname -s) 2023-11-17T05:19:48.488 INFO:tasks.cephadm:Downloading cephadm (repo https://github.com/ceph/ceph ref v16.2.4)... 2023-11-17T05:19:48.489 DEBUG:teuthology.orchestra.run.smithi040:> curl --silent https://raw.githubusercontent.com/ceph/ceph/v16.2.4/src/cephadm/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2023-11-17T05:19:48.706 INFO:teuthology.orchestra.run.smithi040.stdout:-rw-rw-r--. 1 ubuntu ubuntu 292005 Nov 17 05:19 /home/ubuntu/cephtest/cephadm 2023-11-17T05:19:48.707 DEBUG:teuthology.orchestra.run.smithi066:> curl --silent https://raw.githubusercontent.com/ceph/ceph/v16.2.4/src/cephadm/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2023-11-17T05:19:48.847 INFO:teuthology.orchestra.run.smithi066.stdout:-rw-rw-r--. 1 ubuntu ubuntu 292005 Nov 17 05:19 /home/ubuntu/cephtest/cephadm 2023-11-17T05:19:48.848 DEBUG:teuthology.orchestra.run.smithi040:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2023-11-17T05:19:48.873 DEBUG:teuthology.orchestra.run.smithi066:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2023-11-17T05:19:48.940 DEBUG:teuthology.orchestra.run.smithi040:> sudo mkdir -p /etc/ceph 2023-11-17T05:19:48.980 DEBUG:teuthology.orchestra.run.smithi066:> sudo mkdir -p /etc/ceph 2023-11-17T05:19:49.012 DEBUG:teuthology.orchestra.run.smithi040:> sudo chmod 777 /etc/ceph 2023-11-17T05:19:49.054 DEBUG:teuthology.orchestra.run.smithi066:> sudo chmod 777 /etc/ceph 2023-11-17T05:19:49.084 INFO:tasks.cephadm:Writing seed config... 2023-11-17T05:19:49.085 INFO:tasks.cephadm: override: [client] client mount timeout = 600 2023-11-17T05:19:49.085 INFO:tasks.cephadm: override: [client] debug client = 20 2023-11-17T05:19:49.085 INFO:tasks.cephadm: override: [client] debug ms = 1 2023-11-17T05:19:49.085 INFO:tasks.cephadm: override: [client] rados mon op timeout = 900 2023-11-17T05:19:49.086 INFO:tasks.cephadm: override: [client] rados osd op timeout = 900 2023-11-17T05:19:49.086 INFO:tasks.cephadm: override: [global] mon pg warn min per osd = 0 2023-11-17T05:19:49.086 INFO:tasks.cephadm: override: [mds] debug mds = 20 2023-11-17T05:19:49.086 INFO:tasks.cephadm: override: [mds] debug ms = 1 2023-11-17T05:19:49.086 INFO:tasks.cephadm: override: [mds] mds debug frag = True 2023-11-17T05:19:49.086 INFO:tasks.cephadm: override: [mds] mds debug scatterstat = True 2023-11-17T05:19:49.086 INFO:tasks.cephadm: override: [mds] mds op complaint time = 180 2023-11-17T05:19:49.086 INFO:tasks.cephadm: override: [mds] mds verify scatter = True 2023-11-17T05:19:49.086 INFO:tasks.cephadm: override: [mds] osd op complaint time = 180 2023-11-17T05:19:49.086 INFO:tasks.cephadm: override: [mds] rados mon op timeout = 900 2023-11-17T05:19:49.086 INFO:tasks.cephadm: override: [mds] rados osd op timeout = 900 2023-11-17T05:19:49.086 INFO:tasks.cephadm: override: [mgr] debug mgr = 20 2023-11-17T05:19:49.086 INFO:tasks.cephadm: override: [mgr] debug ms = 1 2023-11-17T05:19:49.087 INFO:tasks.cephadm: override: [mon] debug mon = 20 2023-11-17T05:19:49.087 INFO:tasks.cephadm: override: [mon] debug ms = 1 2023-11-17T05:19:49.087 INFO:tasks.cephadm: override: [mon] debug paxos = 20 2023-11-17T05:19:49.087 INFO:tasks.cephadm: override: [mon] mon op complaint time = 120 2023-11-17T05:19:49.087 INFO:tasks.cephadm: override: [osd] bdev async discard = True 2023-11-17T05:19:49.087 INFO:tasks.cephadm: override: [osd] bdev enable discard = True 2023-11-17T05:19:49.087 INFO:tasks.cephadm: override: [osd] bluestore allocator = bitmap 2023-11-17T05:19:49.087 INFO:tasks.cephadm: override: [osd] bluestore block size = 96636764160 2023-11-17T05:19:49.087 INFO:tasks.cephadm: override: [osd] bluestore fsck on mount = True 2023-11-17T05:19:49.087 INFO:tasks.cephadm: override: [osd] debug bluefs = 1/20 2023-11-17T05:19:49.087 INFO:tasks.cephadm: override: [osd] debug bluestore = 1/20 2023-11-17T05:19:49.087 INFO:tasks.cephadm: override: [osd] debug ms = 1 2023-11-17T05:19:49.087 INFO:tasks.cephadm: override: [osd] debug osd = 20 2023-11-17T05:19:49.088 INFO:tasks.cephadm: override: [osd] debug rocksdb = 4/10 2023-11-17T05:19:49.088 INFO:tasks.cephadm: override: [osd] mon osd backfillfull_ratio = 0.85 2023-11-17T05:19:49.088 INFO:tasks.cephadm: override: [osd] mon osd full ratio = 0.9 2023-11-17T05:19:49.088 INFO:tasks.cephadm: override: [osd] mon osd nearfull ratio = 0.8 2023-11-17T05:19:49.088 INFO:tasks.cephadm: override: [osd] osd failsafe full ratio = 0.95 2023-11-17T05:19:49.088 INFO:tasks.cephadm: override: [osd] osd objectstore = bluestore 2023-11-17T05:19:49.088 INFO:tasks.cephadm: override: [osd] osd op complaint time = 180 2023-11-17T05:19:49.089 DEBUG:teuthology.orchestra.run.smithi040:> set -ex 2023-11-17T05:19:49.089 DEBUG:teuthology.orchestra.run.smithi040:> dd of=/home/ubuntu/cephtest/seed.ceph.conf 2023-11-17T05:19:49.120 DEBUG:tasks.cephadm:Final config: [global] # make logging friendly to teuthology log_to_file = true log_to_stderr = false mon cluster log file level = debug mon clock drift allowed = 1.000 # replicate across OSDs, not hosts osd crush chooseleaf type = 0 #osd pool default size = 2 osd pool default erasure code profile = plugin=jerasure technique=reed_sol_van k=2 m=1 ruleset-failure-domain=osd crush-failure-domain=osd # enable some debugging auth debug = true ms die on old message = true ms die on bug = true debug asserts on shutdown = true # adjust warnings mon max pg per osd = 10000# >= luminous mon pg warn max object skew = 0 mon osd allow primary affinity = true mon osd allow pg remap = true mon warn on legacy crush tunables = false mon warn on crush straw calc version zero = false mon warn on no sortbitwise = false mon warn on osd down out interval zero = false mon warn on too few osds = false mon_warn_on_pool_pg_num_not_power_of_two = false # disable pg_autoscaler by default for new pools osd_pool_default_pg_autoscale_mode = off # tests delete pools mon allow pool delete = true fsid = ed6470e2-8508-11ee-8dbe-212e2dc638e7 mon pg warn min per osd = 0 [osd] osd scrub load threshold = 5.0 osd scrub max interval = 600 osd recover clone overlap = true osd recovery max chunk = 1048576 osd deep scrub update digest min age = 30 osd map max advance = 10 osd memory target autotune = true # debugging osd debug shutdown = true osd debug op order = true osd debug verify stray on activate = true osd debug pg log writeout = true osd debug verify cached snaps = true osd debug verify missing on start = true osd debug misdirected ops = true osd op queue = debug_random osd op queue cut off = debug_random osd shutdown pgref assert = true bdev debug aio = true osd sloppy crc = true bdev async discard = True bdev enable discard = True bluestore allocator = bitmap bluestore block size = 96636764160 bluestore fsck on mount = True debug bluefs = 1/20 debug bluestore = 1/20 debug ms = 1 debug osd = 20 debug rocksdb = 4/10 mon osd backfillfull_ratio = 0.85 mon osd full ratio = 0.9 mon osd nearfull ratio = 0.8 osd failsafe full ratio = 0.95 osd objectstore = bluestore osd op complaint time = 180 [mgr] mon reweight min pgs per osd = 4 mon reweight min bytes per osd = 10 mgr/telemetry/nag = false debug mgr = 20 debug ms = 1 [mon] mon data avail warn = 5 mon mgr mkfs grace = 240 mon reweight min pgs per osd = 4 mon osd reporter subtree level = osd mon osd prime pg temp = true mon reweight min bytes per osd = 10 # rotate auth tickets quickly to exercise renewal paths auth mon ticket ttl = 660# 11m auth service ticket ttl = 240# 4m # don't complain about global id reclaim mon_warn_on_insecure_global_id_reclaim = false mon_warn_on_insecure_global_id_reclaim_allowed = false debug mon = 20 debug ms = 1 debug paxos = 20 mon op complaint time = 120 [client.rgw] rgw cache enabled = true rgw enable ops log = true rgw enable usage log = true [client] client mount timeout = 600 debug client = 20 debug ms = 1 rados mon op timeout = 900 rados osd op timeout = 900 [mds] debug mds = 20 debug ms = 1 mds debug frag = True mds debug scatterstat = True mds op complaint time = 180 mds verify scatter = True osd op complaint time = 180 rados mon op timeout = 900 rados osd op timeout = 900 2023-11-17T05:19:49.121 DEBUG:teuthology.orchestra.run.smithi040:mon.smithi040> sudo journalctl -f -n 0 -u ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7@mon.smithi040.service 2023-11-17T05:19:49.163 INFO:tasks.cephadm:Bootstrapping... 2023-11-17T05:19:49.163 DEBUG:teuthology.orchestra.run.smithi040:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.4 -v bootstrap --fsid ed6470e2-8508-11ee-8dbe-212e2dc638e7 --config /home/ubuntu/cephtest/seed.ceph.conf --output-config /etc/ceph/ceph.conf --output-keyring /etc/ceph/ceph.client.admin.keyring --output-pub-ssh-key /home/ubuntu/cephtest/ceph.pub --mon-ip 172.21.15.40 && sudo chmod +r /etc/ceph/ceph.client.admin.keyring 2023-11-17T05:19:49.198 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:-- Logs begin at Fri 2023-11-17 05:02:35 UTC. -- 2023-11-17T05:19:49.468 INFO:teuthology.orchestra.run.smithi040.stderr:Running command: /bin/podman version --format {{.Client.Version}} 2023-11-17T05:19:50.271 INFO:teuthology.orchestra.run.smithi040.stderr:/bin/podman: 4.6.1 2023-11-17T05:19:50.281 INFO:teuthology.orchestra.run.smithi040.stderr:Verifying podman|docker is present... 2023-11-17T05:19:50.281 INFO:teuthology.orchestra.run.smithi040.stderr:Running command: /bin/podman version --format {{.Client.Version}} 2023-11-17T05:19:50.306 INFO:teuthology.orchestra.run.smithi040.stderr:/bin/podman: 4.6.1 2023-11-17T05:19:50.319 INFO:teuthology.orchestra.run.smithi040.stderr:Verifying lvm2 is present... 2023-11-17T05:19:50.319 INFO:teuthology.orchestra.run.smithi040.stderr:Verifying time synchronization is in place... 2023-11-17T05:19:50.320 INFO:teuthology.orchestra.run.smithi040.stderr:Running command: systemctl is-enabled chrony.service 2023-11-17T05:19:50.327 INFO:teuthology.orchestra.run.smithi040.stderr:systemctl: Failed to get unit file state for chrony.service: No such file or directory 2023-11-17T05:19:50.328 INFO:teuthology.orchestra.run.smithi040.stderr:Running command: systemctl is-active chrony.service 2023-11-17T05:19:50.334 INFO:teuthology.orchestra.run.smithi040.stderr:systemctl: inactive 2023-11-17T05:19:50.334 INFO:teuthology.orchestra.run.smithi040.stderr:Running command: systemctl is-enabled chronyd.service 2023-11-17T05:19:50.341 INFO:teuthology.orchestra.run.smithi040.stderr:systemctl: enabled 2023-11-17T05:19:50.342 INFO:teuthology.orchestra.run.smithi040.stderr:Running command: systemctl is-active chronyd.service 2023-11-17T05:19:50.347 INFO:teuthology.orchestra.run.smithi040.stderr:systemctl: active 2023-11-17T05:19:50.348 INFO:teuthology.orchestra.run.smithi040.stderr:Unit chronyd.service is enabled and running 2023-11-17T05:19:50.348 INFO:teuthology.orchestra.run.smithi040.stderr:Repeating the final host check... 2023-11-17T05:19:50.349 INFO:teuthology.orchestra.run.smithi040.stderr:Running command: /bin/podman version --format {{.Client.Version}} 2023-11-17T05:19:50.370 INFO:teuthology.orchestra.run.smithi040.stderr:/bin/podman: 4.6.1 2023-11-17T05:19:50.377 INFO:teuthology.orchestra.run.smithi040.stderr:podman|docker (/bin/podman) is present 2023-11-17T05:19:50.377 INFO:teuthology.orchestra.run.smithi040.stderr:systemctl is present 2023-11-17T05:19:50.377 INFO:teuthology.orchestra.run.smithi040.stderr:lvcreate is present 2023-11-17T05:19:50.377 INFO:teuthology.orchestra.run.smithi040.stderr:Running command: systemctl is-enabled chrony.service 2023-11-17T05:19:50.383 INFO:teuthology.orchestra.run.smithi040.stderr:systemctl: Failed to get unit file state for chrony.service: No such file or directory 2023-11-17T05:19:50.384 INFO:teuthology.orchestra.run.smithi040.stderr:Running command: systemctl is-active chrony.service 2023-11-17T05:19:50.390 INFO:teuthology.orchestra.run.smithi040.stderr:systemctl: inactive 2023-11-17T05:19:50.390 INFO:teuthology.orchestra.run.smithi040.stderr:Running command: systemctl is-enabled chronyd.service 2023-11-17T05:19:50.397 INFO:teuthology.orchestra.run.smithi040.stderr:systemctl: enabled 2023-11-17T05:19:50.397 INFO:teuthology.orchestra.run.smithi040.stderr:Running command: systemctl is-active chronyd.service 2023-11-17T05:19:50.402 INFO:teuthology.orchestra.run.smithi040.stderr:systemctl: active 2023-11-17T05:19:50.403 INFO:teuthology.orchestra.run.smithi040.stderr:Unit chronyd.service is enabled and running 2023-11-17T05:19:50.403 INFO:teuthology.orchestra.run.smithi040.stderr:Host looks OK 2023-11-17T05:19:50.403 INFO:teuthology.orchestra.run.smithi040.stderr:Cluster fsid: ed6470e2-8508-11ee-8dbe-212e2dc638e7 2023-11-17T05:19:50.404 INFO:teuthology.orchestra.run.smithi040.stderr:Acquiring lock 139692835310056 on /run/cephadm/ed6470e2-8508-11ee-8dbe-212e2dc638e7.lock 2023-11-17T05:19:50.404 INFO:teuthology.orchestra.run.smithi040.stderr:Lock 139692835310056 acquired on /run/cephadm/ed6470e2-8508-11ee-8dbe-212e2dc638e7.lock 2023-11-17T05:19:50.404 INFO:teuthology.orchestra.run.smithi040.stderr:Verifying IP 172.21.15.40 port 3300 ... 2023-11-17T05:19:50.405 INFO:teuthology.orchestra.run.smithi040.stderr:Verifying IP 172.21.15.40 port 6789 ... 2023-11-17T05:19:50.405 INFO:teuthology.orchestra.run.smithi040.stderr:Base mon IP is 172.21.15.40, final addrv is [v2:172.21.15.40:3300,v1:172.21.15.40:6789] 2023-11-17T05:19:50.405 INFO:teuthology.orchestra.run.smithi040.stderr:Running command: /sbin/ip route ls 2023-11-17T05:19:50.407 INFO:teuthology.orchestra.run.smithi040.stderr:/sbin/ip: default via 172.21.15.254 dev ens1f0 proto dhcp src 172.21.15.40 metric 100 2023-11-17T05:19:50.407 INFO:teuthology.orchestra.run.smithi040.stderr:/sbin/ip: 172.21.0.0/20 dev ens1f0 proto kernel scope link src 172.21.15.40 metric 100 2023-11-17T05:19:50.408 INFO:teuthology.orchestra.run.smithi040.stderr:Running command: /sbin/ip -6 route ls 2023-11-17T05:19:50.410 INFO:teuthology.orchestra.run.smithi040.stderr:/sbin/ip: ::1 dev lo proto kernel metric 256 pref medium 2023-11-17T05:19:50.410 INFO:teuthology.orchestra.run.smithi040.stderr:/sbin/ip: fe80::/64 dev ens1f0 proto kernel metric 256 pref medium 2023-11-17T05:19:50.410 INFO:teuthology.orchestra.run.smithi040.stderr:/sbin/ip: default via fe80::327c:5e00:6487:71e0 dev ens1f0 proto ra metric 1024 expires 1796sec hoplimit 64 pref medium 2023-11-17T05:19:50.411 INFO:teuthology.orchestra.run.smithi040.stderr:Running command: /sbin/ip -6 addr ls 2023-11-17T05:19:50.414 INFO:teuthology.orchestra.run.smithi040.stderr:/sbin/ip: 1: lo: mtu 65536 state UNKNOWN qlen 1000 2023-11-17T05:19:50.414 INFO:teuthology.orchestra.run.smithi040.stderr:/sbin/ip: inet6 ::1/128 scope host 2023-11-17T05:19:50.415 INFO:teuthology.orchestra.run.smithi040.stderr:/sbin/ip: valid_lft forever preferred_lft forever 2023-11-17T05:19:50.415 INFO:teuthology.orchestra.run.smithi040.stderr:/sbin/ip: 3: ens1f0: mtu 1500 state UP qlen 1000 2023-11-17T05:19:50.415 INFO:teuthology.orchestra.run.smithi040.stderr:/sbin/ip: inet6 fe80::ec4:7aff:febd:12cc/64 scope link 2023-11-17T05:19:50.415 INFO:teuthology.orchestra.run.smithi040.stderr:/sbin/ip: valid_lft forever preferred_lft forever 2023-11-17T05:19:50.415 INFO:teuthology.orchestra.run.smithi040.stderr:Mon IP 172.21.15.40 is in CIDR network 172.21.0.0/20 2023-11-17T05:19:50.415 INFO:teuthology.orchestra.run.smithi040.stderr:- internal network (--cluster-network) has not been provided, OSD replication will default to the public_network 2023-11-17T05:19:50.416 INFO:teuthology.orchestra.run.smithi040.stderr:Pulling container image docker.io/ceph/ceph:v16.2.4... 2023-11-17T05:19:50.416 INFO:teuthology.orchestra.run.smithi040.stderr:Running command: /bin/podman pull docker.io/ceph/ceph:v16.2.4 2023-11-17T05:19:50.514 INFO:teuthology.orchestra.run.smithi040.stderr:/bin/podman: Trying to pull docker.io/ceph/ceph:v16.2.4... 2023-11-17T05:19:57.012 INFO:teuthology.orchestra.run.smithi040.stderr:/bin/podman: Getting image source signatures 2023-11-17T05:19:57.021 INFO:teuthology.orchestra.run.smithi040.stderr:/bin/podman: Copying blob sha256:551101690791055467b1840be133b2e6b029d263771bc6c46a96bb7acdacac39 2023-11-17T05:19:57.021 INFO:teuthology.orchestra.run.smithi040.stderr:/bin/podman: Copying blob sha256:7a0437f04f83f084b7ed68ad9c4a4947e12fc4e1b006b38129bac89114ec3621 2023-11-17T05:20:09.774 INFO:teuthology.orchestra.run.smithi040.stderr:/bin/podman: Copying config sha256:8d91d370c2b86c07de46146aba8d36718eaefa69b1880c77fa312fda6efd7d29 2023-11-17T05:21:24.443 INFO:teuthology.orchestra.run.smithi040.stderr:/bin/podman: Writing manifest to image destination 2023-11-17T05:21:24.444 INFO:teuthology.orchestra.run.smithi040.stderr:/bin/podman: 8d91d370c2b86c07de46146aba8d36718eaefa69b1880c77fa312fda6efd7d29 2023-11-17T05:21:24.444 INFO:teuthology.orchestra.run.smithi040.stderr:Running command: /bin/podman run --rm --ipc=host --no-hosts --net=host --entrypoint ceph --init -e CONTAINER_IMAGE=docker.io/ceph/ceph:v16.2.4 -e NODE_NAME=smithi040 -e CEPH_USE_RANDOM_NONCE=1 docker.io/ceph/ceph:v16.2.4 --version 2023-11-17T05:21:24.444 INFO:teuthology.orchestra.run.smithi040.stderr:ceph: ceph version 16.2.4 (3cbe25cde3cfa028984618ad32de9edc4c1eaed0) pacific (stable) 2023-11-17T05:21:24.444 INFO:teuthology.orchestra.run.smithi040.stderr:Ceph version: ceph version 16.2.4 (3cbe25cde3cfa028984618ad32de9edc4c1eaed0) pacific (stable) 2023-11-17T05:21:24.444 INFO:teuthology.orchestra.run.smithi040.stderr:Extracting ceph user uid/gid from container image... 2023-11-17T05:21:24.444 INFO:teuthology.orchestra.run.smithi040.stderr:Running command: /bin/podman run --rm --ipc=host --no-hosts --net=host --entrypoint stat --init -e CONTAINER_IMAGE=docker.io/ceph/ceph:v16.2.4 -e NODE_NAME=smithi040 -e CEPH_USE_RANDOM_NONCE=1 docker.io/ceph/ceph:v16.2.4 -c %u %g /var/lib/ceph 2023-11-17T05:21:24.444 INFO:teuthology.orchestra.run.smithi040.stderr:stat: 167 167 2023-11-17T05:21:24.444 INFO:teuthology.orchestra.run.smithi040.stderr:Creating initial keys... 2023-11-17T05:21:24.444 INFO:teuthology.orchestra.run.smithi040.stderr:Running command: /bin/podman run --rm --ipc=host --no-hosts --net=host --entrypoint /usr/bin/ceph-authtool --init -e CONTAINER_IMAGE=docker.io/ceph/ceph:v16.2.4 -e NODE_NAME=smithi040 -e CEPH_USE_RANDOM_NONCE=1 docker.io/ceph/ceph:v16.2.4 --gen-print-key 2023-11-17T05:21:24.444 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph-authtool: AQAM+FZlwwg7AxAAlUnuUgrMFszgqF0yUMUX0g== 2023-11-17T05:21:24.444 INFO:teuthology.orchestra.run.smithi040.stderr:Running command: /bin/podman run --rm --ipc=host --no-hosts --net=host --entrypoint /usr/bin/ceph-authtool --init -e CONTAINER_IMAGE=docker.io/ceph/ceph:v16.2.4 -e NODE_NAME=smithi040 -e CEPH_USE_RANDOM_NONCE=1 docker.io/ceph/ceph:v16.2.4 --gen-print-key 2023-11-17T05:21:24.444 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph-authtool: AQAM+FZllg+lKBAAbh0CRD3ULu+3G5vx33YIiA== 2023-11-17T05:21:24.445 INFO:teuthology.orchestra.run.smithi040.stderr:Running command: /bin/podman run --rm --ipc=host --no-hosts --net=host --entrypoint /usr/bin/ceph-authtool --init -e CONTAINER_IMAGE=docker.io/ceph/ceph:v16.2.4 -e NODE_NAME=smithi040 -e CEPH_USE_RANDOM_NONCE=1 docker.io/ceph/ceph:v16.2.4 --gen-print-key 2023-11-17T05:21:24.445 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph-authtool: AQAN+FZlZHMkEhAAl5JiJcyAdcAw/w13/pSMew== 2023-11-17T05:21:24.445 INFO:teuthology.orchestra.run.smithi040.stderr:Creating initial monmap... 2023-11-17T05:21:24.445 INFO:teuthology.orchestra.run.smithi040.stderr:Running command: /bin/podman run --rm --ipc=host --no-hosts --net=host --entrypoint /usr/bin/monmaptool --init -e CONTAINER_IMAGE=docker.io/ceph/ceph:v16.2.4 -e NODE_NAME=smithi040 -e CEPH_USE_RANDOM_NONCE=1 -v /tmp/ceph-tmp454b_8mc:/tmp/monmap:z docker.io/ceph/ceph:v16.2.4 --create --clobber --fsid ed6470e2-8508-11ee-8dbe-212e2dc638e7 --addv smithi040 [v2:172.21.15.40:3300,v1:172.21.15.40:6789] /tmp/monmap 2023-11-17T05:21:24.445 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/monmaptool: /usr/bin/monmaptool: monmap file /tmp/monmap 2023-11-17T05:21:24.445 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/monmaptool: /usr/bin/monmaptool: set fsid to ed6470e2-8508-11ee-8dbe-212e2dc638e7 2023-11-17T05:21:24.445 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/monmaptool: /usr/bin/monmaptool: writing epoch 0 to /tmp/monmap (1 monitors) 2023-11-17T05:21:24.445 INFO:teuthology.orchestra.run.smithi040.stderr:monmaptool for smithi040 [v2:172.21.15.40:3300,v1:172.21.15.40:6789] on /usr/bin/monmaptool: monmap file /tmp/monmap 2023-11-17T05:21:24.445 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/monmaptool: set fsid to ed6470e2-8508-11ee-8dbe-212e2dc638e7 2023-11-17T05:21:24.445 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/monmaptool: writing epoch 0 to /tmp/monmap (1 monitors) 2023-11-17T05:21:24.445 INFO:teuthology.orchestra.run.smithi040.stderr: 2023-11-17T05:21:24.445 INFO:teuthology.orchestra.run.smithi040.stderr:Creating mon... 2023-11-17T05:21:24.445 INFO:teuthology.orchestra.run.smithi040.stderr:Running command: /bin/podman run --rm --ipc=host --no-hosts --net=host --entrypoint /usr/bin/ceph-mon --init -e CONTAINER_IMAGE=docker.io/ceph/ceph:v16.2.4 -e NODE_NAME=smithi040 -e CEPH_USE_RANDOM_NONCE=1 -v /var/log/ceph/ed6470e2-8508-11ee-8dbe-212e2dc638e7:/var/log/ceph:z -v /var/lib/ceph/ed6470e2-8508-11ee-8dbe-212e2dc638e7/mon.smithi040:/var/lib/ceph/mon/ceph-smithi040:z -v /tmp/ceph-tmp5pr_zmcv:/tmp/keyring:z -v /tmp/ceph-tmp454b_8mc:/tmp/monmap:z docker.io/ceph/ceph:v16.2.4 --mkfs -i smithi040 --fsid ed6470e2-8508-11ee-8dbe-212e2dc638e7 -c /dev/null --monmap /tmp/monmap --keyring /tmp/keyring --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-stderr=true 2023-11-17T05:21:24.445 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph-mon: debug 2023-11-17T05:20:14.562+0000 7fe05eb45700 0 set uid:gid to 167:167 (ceph:ceph) 2023-11-17T05:21:24.446 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph-mon: debug 2023-11-17T05:20:14.562+0000 7fe05eb45700 1 imported monmap: 2023-11-17T05:21:24.446 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph-mon: epoch 0 2023-11-17T05:21:24.446 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph-mon: fsid ed6470e2-8508-11ee-8dbe-212e2dc638e7 2023-11-17T05:21:24.446 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph-mon: last_changed 2023-11-17T05:20:13.934038+0000 2023-11-17T05:21:24.446 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph-mon: created 2023-11-17T05:20:13.934038+0000 2023-11-17T05:21:24.446 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph-mon: min_mon_release 0 (unknown) 2023-11-17T05:21:24.446 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph-mon: election_strategy: 1 2023-11-17T05:21:24.446 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph-mon: 0: [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] mon.smithi040 2023-11-17T05:21:24.446 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph-mon: 2023-11-17T05:21:24.446 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph-mon: debug 2023-11-17T05:20:14.562+0000 7fe05eb45700 0 /usr/bin/ceph-mon: set fsid to ed6470e2-8508-11ee-8dbe-212e2dc638e7 2023-11-17T05:21:24.446 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph-mon: debug 2023-11-17T05:20:14.563+0000 7fe05eb45700 4 rocksdb: RocksDB version: 6.8.1 2023-11-17T05:21:24.446 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph-mon: 2023-11-17T05:21:24.446 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph-mon: debug 2023-11-17T05:20:14.563+0000 7fe05eb45700 4 rocksdb: Git sha rocksdb_build_git_sha:@0@ 2023-11-17T05:21:24.447 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph-mon: debug 2023-11-17T05:20:14.563+0000 7fe05eb45700 4 rocksdb: Compile date May 13 2021 2023-11-17T05:21:24.447 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph-mon: debug 2023-11-17T05:20:14.563+0000 7fe05eb45700 4 rocksdb: DB SUMMARY 2023-11-17T05:21:24.447 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph-mon: 2023-11-17T05:21:24.447 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph-mon: debug 2023-11-17T05:20:14.563+0000 7fe05eb45700 4 rocksdb: SST files in /var/lib/ceph/mon/ceph-smithi040/store.db dir, Total Num: 0, files: 2023-11-17T05:21:24.447 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph-mon: 2023-11-17T05:21:24.447 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph-mon: debug 2023-11-17T05:20:14.563+0000 7fe05eb45700 4 rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-smithi040/store.db: 2023-11-17T05:21:24.447 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph-mon: 2023-11-17T05:21:24.447 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph-mon: debug 2023-11-17T05:20:14.563+0000 7fe05eb45700 4 rocksdb: Options.error_if_exists: 0 2023-11-17T05:21:24.447 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph-mon: debug 2023-11-17T05:20:14.563+0000 7fe05eb45700 4 rocksdb: Options.create_if_missing: 1 2023-11-17T05:21:24.447 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph-mon: debug 2023-11-17T05:20:14.563+0000 7fe05eb45700 4 rocksdb: Options.paranoid_checks: 1 2023-11-17T05:21:24.447 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph-mon: debug 2023-11-17T05:20:14.563+0000 7fe05eb45700 4 rocksdb: Options.env: 0x555da0c3d1c0 2023-11-17T05:21:24.447 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph-mon: debug 2023-11-17T05:20:14.563+0000 7fe05eb45700 4 rocksdb: Options.fs: Posix File System 2023-11-17T05:21:24.447 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph-mon: debug 2023-11-17T05:20:14.563+0000 7fe05eb45700 4 rocksdb: Options.info_log: 0x555da2dd4ec0 2023-11-17T05:21:24.447 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph-mon: debug 2023-11-17T05:20:14.563+0000 7fe05eb45700 4 rocksdb: Options.max_file_opening_threads: 16 2023-11-17T05:21:24.448 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph-mon: debug 2023-11-17T05:20:14.563+0000 7fe05eb45700 4 rocksdb: Options.statistics: (nil) 2023-11-17T05:21:24.448 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph-mon: debug 2023-11-17T05:20:14.563+0000 7fe05eb45700 4 rocksdb: Options.use_fsync: 0 2023-11-17T05:21:24.448 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph-mon: debug 2023-11-17T05:20:14.563+0000 7fe05eb45700 4 rocksdb: Options.max_log_file_size: 0 2023-11-17T05:21:24.448 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph-mon: debug 2023-11-17T05:20:14.563+0000 7fe05eb45700 4 rocksdb: Options.max_manifest_file_size: 1073741824 2023-11-17T05:21:24.448 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph-mon: debug 2023-11-17T05:20:14.563+0000 7fe05eb45700 4 rocksdb: Options.log_file_time_to_roll: 0 2023-11-17T05:21:24.448 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph-mon: debug 2023-11-17T05:20:14.563+0000 7fe05eb45700 4 rocksdb: Options.keep_log_file_num: 1000 2023-11-17T05:21:24.448 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph-mon: debug 2023-11-17T05:20:14.563+0000 7fe05eb45700 4 rocksdb: Options.recycle_log_file_num: 0 2023-11-17T05:21:24.448 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph-mon: debug 2023-11-17T05:20:14.563+0000 7fe05eb45700 4 rocksdb: Options.allow_fallocate: 1 2023-11-17T05:21:24.448 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph-mon: debug 2023-11-17T05:20:14.563+0000 7fe05eb45700 4 rocksdb: Options.allow_mmap_reads: 0 2023-11-17T05:21:24.448 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph-mon: debug 2023-11-17T05:20:14.563+0000 7fe05eb45700 4 rocksdb: Options.allow_mmap_writes: 0 2023-11-17T05:21:24.448 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph-mon: debug 2023-11-17T05:20:14.563+0000 7fe05eb45700 4 rocksdb: Options.use_direct_reads: 0 2023-11-17T05:21:24.448 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph-mon: debug 2023-11-17T05:20:14.563+0000 7fe05eb45700 4 rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2023-11-17T05:21:24.448 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph-mon: debug 2023-11-17T05:20:14.563+0000 7fe05eb45700 4 rocksdb: Options.create_missing_column_families: 0 2023-11-17T05:21:24.448 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph-mon: debug 2023-11-17T05:20:14.563+0000 7fe05eb45700 4 rocksdb: Options.db_log_dir: 2023-11-17T05:21:24.449 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph-mon: debug 2023-11-17T05:20:14.563+0000 7fe05eb45700 4 rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-smithi040/store.db 2023-11-17T05:21:24.449 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph-mon: debug 2023-11-17T05:20:14.563+0000 7fe05eb45700 4 rocksdb: Options.table_cache_numshardbits: 6 2023-11-17T05:21:24.449 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph-mon: debug 2023-11-17T05:20:14.563+0000 7fe05eb45700 4 rocksdb: Options.max_subcompactions: 1 2023-11-17T05:21:24.449 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph-mon: debug 2023-11-17T05:20:14.563+0000 7fe05eb45700 4 rocksdb: Options.max_background_flushes: -1 2023-11-17T05:21:24.449 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph-mon: debug 2023-11-17T05:20:14.563+0000 7fe05eb45700 4 rocksdb: Options.WAL_ttl_seconds: 0 2023-11-17T05:21:24.449 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph-mon: debug 2023-11-17T05:20:14.563+0000 7fe05eb45700 4 rocksdb: Options.WAL_size_limit_MB: 0 2023-11-17T05:21:24.449 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph-mon: debug 2023-11-17T05:20:14.563+0000 7fe05eb45700 4 rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2023-11-17T05:21:24.449 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph-mon: debug 2023-11-17T05:20:14.563+0000 7fe05eb45700 4 rocksdb: Options.manifest_preallocation_size: 4194304 2023-11-17T05:21:24.449 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph-mon: debug 2023-11-17T05:20:14.563+0000 7fe05eb45700 4 rocksdb: Options.is_fd_close_on_exec: 1 2023-11-17T05:21:24.449 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph-mon: debug 2023-11-17T05:20:14.563+0000 7fe05eb45700 4 rocksdb: Options.advise_random_on_open: 1 2023-11-17T05:21:24.450 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph-mon: debug 2023-11-17T05:20:14.563+0000 7fe05eb45700 4 rocksdb: Options.db_write_buffer_size: 0 2023-11-17T05:21:24.450 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph-mon: debug 2023-11-17T05:20:14.563+0000 7fe05eb45700 4 rocksdb: Options.write_buffer_manager: 0x555da2ddb530 2023-11-17T05:21:24.450 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph-mon: debug 2023-11-17T05:20:14.563+0000 7fe05eb45700 4 rocksdb: Options.access_hint_on_compaction_start: 1 2023-11-17T05:21:24.450 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph-mon: debug 2023-11-17T05:20:14.563+0000 7fe05eb45700 4 rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2023-11-17T05:21:24.450 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph-mon: debug 2023-11-17T05:20:14.563+0000 7fe05eb45700 4 rocksdb: Options.random_access_max_buffer_size: 1048576 2023-11-17T05:21:24.450 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph-mon: debug 2023-11-17T05:20:14.563+0000 7fe05eb45700 4 rocksdb: Options.use_adaptive_mutex: 0 2023-11-17T05:21:24.450 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph-mon: debug 2023-11-17T05:20:14.563+0000 7fe05eb45700 4 rocksdb: Options.rate_limiter: (nil) 2023-11-17T05:21:24.450 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph-mon: debug 2023-11-17T05:20:14.563+0000 7fe05eb45700 4 rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2023-11-17T05:21:24.450 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph-mon: debug 2023-11-17T05:20:14.563+0000 7fe05eb45700 4 rocksdb: Options.wal_recovery_mode: 2 2023-11-17T05:21:24.450 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph-mon: debug 2023-11-17T05:20:14.563+0000 7fe05eb45700 4 rocksdb: Options.enable_thread_tracking: 0 2023-11-17T05:21:24.450 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph-mon: debug 2023-11-17T05:20:14.563+0000 7fe05eb45700 4 rocksdb: Options.enable_pipelined_write: 0 2023-11-17T05:21:24.450 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph-mon: debug 2023-11-17T05:20:14.563+0000 7fe05eb45700 4 rocksdb: Options.unordered_write: 0 2023-11-17T05:21:24.450 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph-mon: debug 2023-11-17T05:20:14.563+0000 7fe05eb45700 4 rocksdb: Options.allow_concurrent_memtable_write: 1 2023-11-17T05:21:24.451 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph-mon: debug 2023-11-17T05:20:14.563+0000 7fe05eb45700 4 rocksdb: Options.enable_write_thread_adaptive_yield: 1 2023-11-17T05:21:24.451 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph-mon: debug 2023-11-17T05:20:14.563+0000 7fe05eb45700 4 rocksdb: Options.write_thread_max_yield_usec: 100 2023-11-17T05:21:24.451 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph-mon: debug 2023-11-17T05:20:14.563+0000 7fe05eb45700 4 rocksdb: Options.write_thread_slow_yield_usec: 3 2023-11-17T05:21:24.451 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph-mon: debug 2023-11-17T05:20:14.563+0000 7fe05eb45700 4 rocksdb: Options.row_cache: None 2023-11-17T05:21:24.451 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph-mon: debug 2023-11-17T05:20:14.563+0000 7fe05eb45700 4 rocksdb: Options.wal_filter: None 2023-11-17T05:21:24.451 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph-mon: debug 2023-11-17T05:20:14.563+0000 7fe05eb45700 4 rocksdb: Options.avoid_flush_during_recovery: 0 2023-11-17T05:21:24.451 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph-mon: debug 2023-11-17T05:20:14.563+0000 7fe05eb45700 4 rocksdb: Options.allow_ingest_behind: 0 2023-11-17T05:21:24.451 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph-mon: debug 2023-11-17T05:20:14.563+0000 7fe05eb45700 4 rocksdb: Options.preserve_deletes: 0 2023-11-17T05:21:24.451 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph-mon: debug 2023-11-17T05:20:14.563+0000 7fe05eb45700 4 rocksdb: Options.two_write_queues: 0 2023-11-17T05:21:24.451 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph-mon: debug 2023-11-17T05:20:14.563+0000 7fe05eb45700 4 rocksdb: Options.manual_wal_flush: 0 2023-11-17T05:21:24.451 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph-mon: debug 2023-11-17T05:20:14.563+0000 7fe05eb45700 4 rocksdb: Options.atomic_flush: 0 2023-11-17T05:21:24.451 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph-mon: debug 2023-11-17T05:20:14.563+0000 7fe05eb45700 4 rocksdb: Options.avoid_unnecessary_blocking_io: 0 2023-11-17T05:21:24.452 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph-mon: debug 2023-11-17T05:20:14.563+0000 7fe05eb45700 4 rocksdb: Options.persist_stats_to_disk: 0 2023-11-17T05:21:24.452 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph-mon: debug 2023-11-17T05:20:14.563+0000 7fe05eb45700 4 rocksdb: Options.write_dbid_to_manifest: 0 2023-11-17T05:21:24.452 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph-mon: debug 2023-11-17T05:20:14.563+0000 7fe05eb45700 4 rocksdb: Options.log_readahead_size: 0 2023-11-17T05:21:24.452 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph-mon: debug 2023-11-17T05:20:14.563+0000 7fe05eb45700 4 rocksdb: Options.sst_file_checksum_func: Unknown 2023-11-17T05:21:24.452 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph-mon: debug 2023-11-17T05:20:14.563+0000 7fe05eb45700 4 rocksdb: Options.max_background_jobs: 2 2023-11-17T05:21:24.452 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph-mon: debug 2023-11-17T05:20:14.563+0000 7fe05eb45700 4 rocksdb: Options.max_background_compactions: -1 2023-11-17T05:21:24.452 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph-mon: debug 2023-11-17T05:20:14.563+0000 7fe05eb45700 4 rocksdb: Options.avoid_flush_during_shutdown: 0 2023-11-17T05:21:24.452 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph-mon: debug 2023-11-17T05:20:14.563+0000 7fe05eb45700 4 rocksdb: Options.writable_file_max_buffer_size: 1048576 2023-11-17T05:21:24.452 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph-mon: debug 2023-11-17T05:20:14.563+0000 7fe05eb45700 4 rocksdb: Options.delayed_write_rate : 16777216 2023-11-17T05:21:24.452 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph-mon: debug 2023-11-17T05:20:14.563+0000 7fe05eb45700 4 rocksdb: Options.max_total_wal_size: 0 2023-11-17T05:21:24.452 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph-mon: debug 2023-11-17T05:20:14.563+0000 7fe05eb45700 4 rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2023-11-17T05:21:24.452 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph-mon: debug 2023-11-17T05:20:14.563+0000 7fe05eb45700 4 rocksdb: Options.stats_dump_period_sec: 600 2023-11-17T05:21:24.452 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph-mon: debug 2023-11-17T05:20:14.563+0000 7fe05eb45700 4 rocksdb: Options.stats_persist_period_sec: 600 2023-11-17T05:21:24.453 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph-mon: debug 2023-11-17T05:20:14.563+0000 7fe05eb45700 4 rocksdb: Options.stats_history_buffer_size: 1048576 2023-11-17T05:21:24.453 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph-mon: debug 2023-11-17T05:20:14.563+0000 7fe05eb45700 4 rocksdb: Options.max_open_files: -1 2023-11-17T05:21:24.453 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph-mon: debug 2023-11-17T05:20:14.563+0000 7fe05eb45700 4 rocksdb: Options.bytes_per_sync: 0 2023-11-17T05:21:24.453 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph-mon: debug 2023-11-17T05:20:14.563+0000 7fe05eb45700 4 rocksdb: Options.wal_bytes_per_sync: 0 2023-11-17T05:21:24.453 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph-mon: debug 2023-11-17T05:20:14.563+0000 7fe05eb45700 4 rocksdb: Options.strict_bytes_per_sync: 0 2023-11-17T05:21:24.453 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph-mon: debug 2023-11-17T05:20:14.563+0000 7fe05eb45700 4 rocksdb: Options.compaction_readahead_size: 0 2023-11-17T05:21:24.453 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph-mon: debug 2023-11-17T05:20:14.563+0000 7fe05eb45700 4 rocksdb: Compression algorithms supported: 2023-11-17T05:21:24.453 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph-mon: debug 2023-11-17T05:20:14.563+0000 7fe05eb45700 4 rocksdb: kZSTDNotFinalCompression supported: 0 2023-11-17T05:21:24.453 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph-mon: debug 2023-11-17T05:20:14.563+0000 7fe05eb45700 4 rocksdb: kZSTD supported: 0 2023-11-17T05:21:24.453 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph-mon: debug 2023-11-17T05:20:14.563+0000 7fe05eb45700 4 rocksdb: kXpressCompression supported: 0 2023-11-17T05:21:24.453 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph-mon: debug 2023-11-17T05:20:14.563+0000 7fe05eb45700 4 rocksdb: kLZ4HCCompression supported: 1 2023-11-17T05:21:24.453 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph-mon: debug 2023-11-17T05:20:14.563+0000 7fe05eb45700 4 rocksdb: kLZ4Compression supported: 1 2023-11-17T05:21:24.453 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph-mon: debug 2023-11-17T05:20:14.563+0000 7fe05eb45700 4 rocksdb: kBZip2Compression supported: 0 2023-11-17T05:21:24.454 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph-mon: debug 2023-11-17T05:20:14.563+0000 7fe05eb45700 4 rocksdb: kZlibCompression supported: 1 2023-11-17T05:21:24.454 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph-mon: debug 2023-11-17T05:20:14.563+0000 7fe05eb45700 4 rocksdb: kSnappyCompression supported: 1 2023-11-17T05:21:24.454 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph-mon: debug 2023-11-17T05:20:14.563+0000 7fe05eb45700 4 rocksdb: Fast CRC32 supported: Supported on x86 2023-11-17T05:21:24.454 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph-mon: debug 2023-11-17T05:20:14.564+0000 7fe05eb45700 4 rocksdb: [db_impl/db_impl_open.cc:273] Creating manifest 1 2023-11-17T05:21:24.454 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph-mon: 2023-11-17T05:21:24.454 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph-mon: debug 2023-11-17T05:20:14.564+0000 7fe05eb45700 4 rocksdb: [version_set.cc:4413] Recovering from manifest file: /var/lib/ceph/mon/ceph-smithi040/store.db/MANIFEST-000001 2023-11-17T05:21:24.454 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph-mon: 2023-11-17T05:21:24.454 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph-mon: debug 2023-11-17T05:20:14.564+0000 7fe05eb45700 4 rocksdb: [column_family.cc:552] --------------- Options for column family [default]: 2023-11-17T05:21:24.454 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph-mon: 2023-11-17T05:21:24.454 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph-mon: debug 2023-11-17T05:20:14.564+0000 7fe05eb45700 4 rocksdb: Options.comparator: leveldb.BytewiseComparator 2023-11-17T05:21:24.454 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph-mon: debug 2023-11-17T05:20:14.564+0000 7fe05eb45700 4 rocksdb: Options.merge_operator: 2023-11-17T05:21:24.454 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph-mon: debug 2023-11-17T05:20:14.564+0000 7fe05eb45700 4 rocksdb: Options.compaction_filter: None 2023-11-17T05:21:24.455 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph-mon: debug 2023-11-17T05:20:14.564+0000 7fe05eb45700 4 rocksdb: Options.compaction_filter_factory: None 2023-11-17T05:21:24.455 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph-mon: debug 2023-11-17T05:20:14.564+0000 7fe05eb45700 4 rocksdb: Options.memtable_factory: SkipListFactory 2023-11-17T05:21:24.455 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph-mon: debug 2023-11-17T05:20:14.564+0000 7fe05eb45700 4 rocksdb: Options.table_factory: BlockBasedTable 2023-11-17T05:21:24.455 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph-mon: debug 2023-11-17T05:20:14.564+0000 7fe05eb45700 4 rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x555da2cec120) 2023-11-17T05:21:24.455 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph-mon: cache_index_and_filter_blocks: 1 2023-11-17T05:21:24.455 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph-mon: cache_index_and_filter_blocks_with_high_priority: 0 2023-11-17T05:21:24.455 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph-mon: pin_l0_filter_and_index_blocks_in_cache: 0 2023-11-17T05:21:24.455 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph-mon: pin_top_level_index_and_filter: 1 2023-11-17T05:21:24.455 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph-mon: index_type: 0 2023-11-17T05:21:24.455 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph-mon: data_block_index_type: 0 2023-11-17T05:21:24.455 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph-mon: index_shortening: 1 2023-11-17T05:21:24.455 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph-mon: data_block_hash_table_util_ratio: 0.750000 2023-11-17T05:21:24.455 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph-mon: hash_index_allow_collision: 1 2023-11-17T05:21:24.456 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph-mon: checksum: 1 2023-11-17T05:21:24.482 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph-mon: no_block_cache: 0 2023-11-17T05:21:24.599 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph-mon: block_cache: 0x555da2d22d10 2023-11-17T05:21:24.599 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph-mon: block_cache_name: BinnedLRUCache 2023-11-17T05:21:24.599 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph-mon: block_cache_options: 2023-11-17T05:21:24.627 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph-mon: capacity : 536870912 2023-11-17T05:21:24.627 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph-mon: num_shard_bits : 4 2023-11-17T05:21:24.627 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph-mon: strict_capacity_limit : 0 2023-11-17T05:21:24.627 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph-mon: high_pri_pool_ratio: 0.000 2023-11-17T05:21:24.627 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph-mon: block_cache_compressed: (nil) 2023-11-17T05:21:24.627 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph-mon: persistent_cache: (nil) 2023-11-17T05:21:24.628 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph-mon: block_size: 4096 2023-11-17T05:21:24.628 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph-mon: block_size_deviation: 10 2023-11-17T05:21:24.628 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph-mon: block_restart_interval: 16 2023-11-17T05:21:24.628 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph-mon: index_block_restart_interval: 1 2023-11-17T05:21:24.628 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph-mon: metadata_block_size: 4096 2023-11-17T05:21:24.628 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph-mon: partition_filters: 0 2023-11-17T05:21:24.628 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph-mon: use_delta_encoding: 1 2023-11-17T05:21:24.628 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph-mon: filter_policy: rocksdb.BuiltinBloomFilter 2023-11-17T05:21:24.628 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph-mon: whole_key_filtering: 1 2023-11-17T05:21:24.628 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph-mon: verify_compression: 0 2023-11-17T05:21:24.628 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph-mon: read_amp_bytes_per_bit: 0 2023-11-17T05:21:24.628 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph-mon: format_version: 2 2023-11-17T05:21:24.628 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph-mon: enable_index_compression: 1 2023-11-17T05:21:24.628 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph-mon: block_align: 0 2023-11-17T05:21:24.629 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph-mon: 2023-11-17T05:21:24.629 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph-mon: debug 2023-11-17T05:20:14.564+0000 7fe05eb45700 4 rocksdb: Options.write_buffer_size: 33554432 2023-11-17T05:21:24.629 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph-mon: debug 2023-11-17T05:20:14.564+0000 7fe05eb45700 4 rocksdb: Options.max_write_buffer_number: 2 2023-11-17T05:21:24.629 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph-mon: debug 2023-11-17T05:20:14.564+0000 7fe05eb45700 4 rocksdb: Options.compression: NoCompression 2023-11-17T05:21:24.629 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph-mon: debug 2023-11-17T05:20:14.564+0000 7fe05eb45700 4 rocksdb: Options.bottommost_compression: Disabled 2023-11-17T05:21:24.629 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph-mon: debug 2023-11-17T05:20:14.564+0000 7fe05eb45700 4 rocksdb: Options.prefix_extractor: nullptr 2023-11-17T05:21:24.629 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph-mon: debug 2023-11-17T05:20:14.564+0000 7fe05eb45700 4 rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2023-11-17T05:21:24.629 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph-mon: debug 2023-11-17T05:20:14.564+0000 7fe05eb45700 4 rocksdb: Options.num_levels: 7 2023-11-17T05:21:24.629 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph-mon: debug 2023-11-17T05:20:14.564+0000 7fe05eb45700 4 rocksdb: Options.min_write_buffer_number_to_merge: 1 2023-11-17T05:21:24.629 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph-mon: debug 2023-11-17T05:20:14.564+0000 7fe05eb45700 4 rocksdb: Options.max_write_buffer_number_to_maintain: 0 2023-11-17T05:21:24.629 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph-mon: debug 2023-11-17T05:20:14.564+0000 7fe05eb45700 4 rocksdb: Options.max_write_buffer_size_to_maintain: 0 2023-11-17T05:21:24.629 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph-mon: debug 2023-11-17T05:20:14.564+0000 7fe05eb45700 4 rocksdb: Options.bottommost_compression_opts.window_bits: -14 2023-11-17T05:21:24.629 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph-mon: debug 2023-11-17T05:20:14.564+0000 7fe05eb45700 4 rocksdb: Options.bottommost_compression_opts.level: 32767 2023-11-17T05:21:24.630 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph-mon: debug 2023-11-17T05:20:14.564+0000 7fe05eb45700 4 rocksdb: Options.bottommost_compression_opts.strategy: 0 2023-11-17T05:21:24.630 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph-mon: debug 2023-11-17T05:20:14.564+0000 7fe05eb45700 4 rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2023-11-17T05:21:24.630 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph-mon: debug 2023-11-17T05:20:14.564+0000 7fe05eb45700 4 rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2023-11-17T05:21:24.630 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph-mon: debug 2023-11-17T05:20:14.564+0000 7fe05eb45700 4 rocksdb: Options.bottommost_compression_opts.enabled: false 2023-11-17T05:21:24.630 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph-mon: debug 2023-11-17T05:20:14.564+0000 7fe05eb45700 4 rocksdb: Options.compression_opts.window_bits: -14 2023-11-17T05:21:24.630 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph-mon: debug 2023-11-17T05:20:14.564+0000 7fe05eb45700 4 rocksdb: Options.compression_opts.level: 32767 2023-11-17T05:21:24.630 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph-mon: debug 2023-11-17T05:20:14.564+0000 7fe05eb45700 4 rocksdb: Options.compression_opts.strategy: 0 2023-11-17T05:21:24.630 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph-mon: debug 2023-11-17T05:20:14.564+0000 7fe05eb45700 4 rocksdb: Options.compression_opts.max_dict_bytes: 0 2023-11-17T05:21:24.631 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph-mon: debug 2023-11-17T05:20:14.564+0000 7fe05eb45700 4 rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2023-11-17T05:21:24.631 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph-mon: debug 2023-11-17T05:20:14.564+0000 7fe05eb45700 4 rocksdb: Options.compression_opts.enabled: false 2023-11-17T05:21:24.631 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph-mon: debug 2023-11-17T05:20:14.564+0000 7fe05eb45700 4 rocksdb: Options.level0_file_num_compaction_trigger: 4 2023-11-17T05:21:24.631 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph-mon: debug 2023-11-17T05:20:14.564+0000 7fe05eb45700 4 rocksdb: Options.level0_slowdown_writes_trigger: 20 2023-11-17T05:21:24.631 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph-mon: debug 2023-11-17T05:20:14.564+0000 7fe05eb45700 4 rocksdb: Options.level0_stop_writes_trigger: 36 2023-11-17T05:21:24.631 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph-mon: debug 2023-11-17T05:20:14.564+0000 7fe05eb45700 4 rocksdb: Options.target_file_size_base: 67108864 2023-11-17T05:21:24.631 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph-mon: debug 2023-11-17T05:20:14.564+0000 7fe05eb45700 4 rocksdb: Options.target_file_size_multiplier: 1 2023-11-17T05:21:24.631 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph-mon: debug 2023-11-17T05:20:14.564+0000 7fe05eb45700 4 rocksdb: Options.max_bytes_for_level_base: 268435456 2023-11-17T05:21:24.631 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph-mon: debug 2023-11-17T05:20:14.564+0000 7fe05eb45700 4 rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2023-11-17T05:21:24.673 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph-mon: debug 2023-11-17T05:20:14.564+0000 7fe05eb45700 4 rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2023-11-17T05:21:24.673 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph-mon: debug 2023-11-17T05:20:14.564+0000 7fe05eb45700 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2023-11-17T05:21:24.877 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph-mon: debug 2023-11-17T05:20:14.564+0000 7fe05eb45700 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2023-11-17T05:21:24.878 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph-mon: debug 2023-11-17T05:20:14.564+0000 7fe05eb45700 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2023-11-17T05:21:24.878 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph-mon: debug 2023-11-17T05:20:14.564+0000 7fe05eb45700 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2023-11-17T05:21:24.878 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph-mon: debug 2023-11-17T05:20:14.564+0000 7fe05eb45700 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2023-11-17T05:21:24.878 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph-mon: debug 2023-11-17T05:20:14.564+0000 7fe05eb45700 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2023-11-17T05:21:24.878 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph-mon: debug 2023-11-17T05:20:14.564+0000 7fe05eb45700 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2023-11-17T05:21:24.878 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph-mon: debug 2023-11-17T05:20:14.564+0000 7fe05eb45700 4 rocksdb: Options.max_sequential_skip_in_iterations: 8 2023-11-17T05:21:24.878 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph-mon: debug 2023-11-17T05:20:14.564+0000 7fe05eb45700 4 rocksdb: Options.max_compaction_bytes: 1677721600 2023-11-17T05:21:24.878 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph-mon: debug 2023-11-17T05:20:14.564+0000 7fe05eb45700 4 rocksdb: Options.arena_block_size: 4194304 2023-11-17T05:21:24.878 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph-mon: debug 2023-11-17T05:20:14.564+0000 7fe05eb45700 4 rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2023-11-17T05:21:24.878 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph-mon: debug 2023-11-17T05:20:14.564+0000 7fe05eb45700 4 rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2023-11-17T05:21:24.878 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph-mon: debug 2023-11-17T05:20:14.564+0000 7fe05eb45700 4 rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2023-11-17T05:21:24.878 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph-mon: debug 2023-11-17T05:20:14.564+0000 7fe05eb45700 4 rocksdb: Options.disable_auto_compactions: 0 2023-11-17T05:21:24.879 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph-mon: debug 2023-11-17T05:20:14.564+0000 7fe05eb45700 4 rocksdb: Options.compaction_style: kCompactionStyleLevel 2023-11-17T05:21:24.879 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph-mon: debug 2023-11-17T05:20:14.564+0000 7fe05eb45700 4 rocksdb: Options.compaction_pri: kMinOverlappingRatio 2023-11-17T05:21:24.879 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph-mon: debug 2023-11-17T05:20:14.564+0000 7fe05eb45700 4 rocksdb: Options.compaction_options_universal.size_ratio: 1 2023-11-17T05:21:24.906 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph-mon: debug 2023-11-17T05:20:14.564+0000 7fe05eb45700 4 rocksdb: Options.compaction_options_universal.min_merge_width: 2 2023-11-17T05:21:24.906 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph-mon: debug 2023-11-17T05:20:14.564+0000 7fe05eb45700 4 rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2023-11-17T05:21:24.906 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph-mon: debug 2023-11-17T05:20:14.564+0000 7fe05eb45700 4 rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2023-11-17T05:21:24.906 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph-mon: debug 2023-11-17T05:20:14.564+0000 7fe05eb45700 4 rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2023-11-17T05:21:24.906 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph-mon: debug 2023-11-17T05:20:14.564+0000 7fe05eb45700 4 rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2023-11-17T05:21:24.906 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph-mon: debug 2023-11-17T05:20:14.564+0000 7fe05eb45700 4 rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2023-11-17T05:21:24.906 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph-mon: debug 2023-11-17T05:20:14.564+0000 7fe05eb45700 4 rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2023-11-17T05:21:24.906 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph-mon: debug 2023-11-17T05:20:14.564+0000 7fe05eb45700 4 rocksdb: Options.table_properties_collectors: 2023-11-17T05:21:24.907 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph-mon: debug 2023-11-17T05:20:14.564+0000 7fe05eb45700 4 rocksdb: Options.inplace_update_support: 0 2023-11-17T05:21:24.907 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph-mon: debug 2023-11-17T05:20:14.564+0000 7fe05eb45700 4 rocksdb: Options.inplace_update_num_locks: 10000 2023-11-17T05:21:24.907 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph-mon: debug 2023-11-17T05:20:14.564+0000 7fe05eb45700 4 rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2023-11-17T05:21:24.907 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph-mon: debug 2023-11-17T05:20:14.564+0000 7fe05eb45700 4 rocksdb: Options.memtable_whole_key_filtering: 0 2023-11-17T05:21:24.907 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph-mon: debug 2023-11-17T05:20:14.564+0000 7fe05eb45700 4 rocksdb: Options.memtable_huge_page_size: 0 2023-11-17T05:21:24.907 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph-mon: debug 2023-11-17T05:20:14.564+0000 7fe05eb45700 4 rocksdb: Options.bloom_locality: 0 2023-11-17T05:21:24.907 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph-mon: debug 2023-11-17T05:20:14.564+0000 7fe05eb45700 4 rocksdb: Options.max_successive_merges: 0 2023-11-17T05:21:24.907 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph-mon: debug 2023-11-17T05:20:14.564+0000 7fe05eb45700 4 rocksdb: Options.optimize_filters_for_hits: 0 2023-11-17T05:21:24.907 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph-mon: debug 2023-11-17T05:20:14.564+0000 7fe05eb45700 4 rocksdb: Options.paranoid_file_checks: 0 2023-11-17T05:21:24.907 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph-mon: debug 2023-11-17T05:20:14.564+0000 7fe05eb45700 4 rocksdb: Options.force_consistency_checks: 0 2023-11-17T05:21:24.907 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph-mon: debug 2023-11-17T05:20:14.564+0000 7fe05eb45700 4 rocksdb: Options.report_bg_io_stats: 0 2023-11-17T05:21:24.907 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph-mon: debug 2023-11-17T05:20:14.564+0000 7fe05eb45700 4 rocksdb: Options.ttl: 2592000 2023-11-17T05:21:24.907 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph-mon: debug 2023-11-17T05:20:14.564+0000 7fe05eb45700 4 rocksdb: Options.periodic_compaction_seconds: 0 2023-11-17T05:21:24.908 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph-mon: debug 2023-11-17T05:20:14.565+0000 7fe05eb45700 4 rocksdb: [version_set.cc:4568] Recovered from manifest file:/var/lib/ceph/mon/ceph-smithi040/store.db/MANIFEST-000001 succeeded,manifest_file_number is 1, next_file_number is 3, last_sequence is 0, log_number is 0,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2023-11-17T05:21:24.908 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph-mon: 2023-11-17T05:21:24.908 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph-mon: debug 2023-11-17T05:20:14.565+0000 7fe05eb45700 4 rocksdb: [version_set.cc:4577] Column family [default] (ID 0), log number is 0 2023-11-17T05:21:24.908 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph-mon: 2023-11-17T05:21:24.908 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph-mon: debug 2023-11-17T05:20:14.566+0000 7fe05eb45700 4 rocksdb: DB pointer 0x555da2de7800 2023-11-17T05:21:24.908 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph-mon: debug 2023-11-17T05:20:14.566+0000 7fe0479de700 4 rocksdb: [db_impl/db_impl.cc:850] ------- DUMPING STATS ------- 2023-11-17T05:21:24.908 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph-mon: debug 2023-11-17T05:20:14.566+0000 7fe0479de700 4 rocksdb: [db_impl/db_impl.cc:851] 2023-11-17T05:21:24.908 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph-mon: ** DB Stats ** 2023-11-17T05:21:24.908 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph-mon: Uptime(secs): 0.0 total, 0.0 interval 2023-11-17T05:21:24.908 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph-mon: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2023-11-17T05:21:24.908 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph-mon: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2023-11-17T05:21:24.982 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph-mon: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2023-11-17T05:21:25.099 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph-mon: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2023-11-17T05:21:25.099 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph-mon: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2023-11-17T05:21:25.099 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph-mon: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2023-11-17T05:21:25.099 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph-mon: 2023-11-17T05:21:25.099 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph-mon: ** Compaction Stats [default] ** 2023-11-17T05:21:25.099 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph-mon: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-11-17T05:21:25.100 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph-mon: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-11-17T05:21:25.100 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph-mon: Sum 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2023-11-17T05:21:25.100 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph-mon: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2023-11-17T05:21:25.100 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph-mon: 2023-11-17T05:21:25.100 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph-mon: ** Compaction Stats [default] ** 2023-11-17T05:21:25.100 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph-mon: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-11-17T05:21:25.123 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph-mon: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-11-17T05:21:25.123 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph-mon: Uptime(secs): 0.0 total, 0.0 interval 2023-11-17T05:21:25.123 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph-mon: Flush(GB): cumulative 0.000, interval 0.000 2023-11-17T05:21:25.123 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph-mon: AddFile(GB): cumulative 0.000, interval 0.000 2023-11-17T05:21:25.123 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph-mon: AddFile(Total Files): cumulative 0, interval 0 2023-11-17T05:21:25.123 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph-mon: AddFile(L0 Files): cumulative 0, interval 0 2023-11-17T05:21:25.123 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph-mon: AddFile(Keys): cumulative 0, interval 0 2023-11-17T05:21:25.123 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph-mon: Cumulative compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-11-17T05:21:25.123 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph-mon: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-11-17T05:21:25.123 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph-mon: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-11-17T05:21:25.123 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph-mon: 2023-11-17T05:21:25.123 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph-mon: ** File Read Latency Histogram By Level [default] ** 2023-11-17T05:21:25.124 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph-mon: 2023-11-17T05:21:25.124 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph-mon: ** Compaction Stats [default] ** 2023-11-17T05:21:25.124 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph-mon: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-11-17T05:21:25.124 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph-mon: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-11-17T05:21:25.124 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph-mon: Sum 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2023-11-17T05:21:25.124 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph-mon: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2023-11-17T05:21:25.124 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph-mon: 2023-11-17T05:21:25.124 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph-mon: ** Compaction Stats [default] ** 2023-11-17T05:21:25.124 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph-mon: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-11-17T05:21:25.124 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph-mon: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-11-17T05:21:25.124 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph-mon: Uptime(secs): 0.0 total, 0.0 interval 2023-11-17T05:21:25.124 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph-mon: Flush(GB): cumulative 0.000, interval 0.000 2023-11-17T05:21:25.124 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph-mon: AddFile(GB): cumulative 0.000, interval 0.000 2023-11-17T05:21:25.125 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph-mon: AddFile(Total Files): cumulative 0, interval 0 2023-11-17T05:21:25.125 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph-mon: AddFile(L0 Files): cumulative 0, interval 0 2023-11-17T05:21:25.125 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph-mon: AddFile(Keys): cumulative 0, interval 0 2023-11-17T05:21:25.125 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph-mon: Cumulative compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-11-17T05:21:25.125 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph-mon: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-11-17T05:21:25.125 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph-mon: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-11-17T05:21:25.125 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph-mon: 2023-11-17T05:21:25.125 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph-mon: ** File Read Latency Histogram By Level [default] ** 2023-11-17T05:21:25.125 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph-mon: 2023-11-17T05:21:25.125 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph-mon: debug 2023-11-17T05:20:14.568+0000 7fe05eb45700 4 rocksdb: [db_impl/db_impl.cc:397] Shutdown: canceling all background work 2023-11-17T05:21:25.125 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph-mon: debug 2023-11-17T05:20:14.568+0000 7fe05eb45700 4 rocksdb: [db_impl/db_impl.cc:573] Shutdown complete 2023-11-17T05:21:25.125 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph-mon: debug 2023-11-17T05:20:14.568+0000 7fe05eb45700 0 /usr/bin/ceph-mon: created monfs at /var/lib/ceph/mon/ceph-smithi040 for mon.smithi040 2023-11-17T05:21:25.125 INFO:teuthology.orchestra.run.smithi040.stderr:create mon.smithi040 on 2023-11-17T05:21:25.126 INFO:teuthology.orchestra.run.smithi040.stderr:Running command: install -d -m0770 -o 167 -g 167 /var/run/ceph/ed6470e2-8508-11ee-8dbe-212e2dc638e7 2023-11-17T05:21:25.126 INFO:teuthology.orchestra.run.smithi040.stderr:Running command: systemctl disable ceph.target 2023-11-17T05:21:25.126 INFO:teuthology.orchestra.run.smithi040.stderr:systemctl: Removed /etc/systemd/system/multi-user.target.wants/ceph.target. 2023-11-17T05:21:25.126 INFO:teuthology.orchestra.run.smithi040.stderr:Running command: systemctl enable ceph.target 2023-11-17T05:21:25.126 INFO:teuthology.orchestra.run.smithi040.stderr:systemctl: Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target → /etc/systemd/system/ceph.target. 2023-11-17T05:21:25.126 INFO:teuthology.orchestra.run.smithi040.stderr:Running command: systemctl start ceph.target 2023-11-17T05:21:25.126 INFO:teuthology.orchestra.run.smithi040.stderr:Running command: systemctl enable ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7.target 2023-11-17T05:21:25.126 INFO:teuthology.orchestra.run.smithi040.stderr:systemctl: Created symlink /etc/systemd/system/multi-user.target.wants/ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7.target → /etc/systemd/system/ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7.target. 2023-11-17T05:21:25.126 INFO:teuthology.orchestra.run.smithi040.stderr:systemctl: Created symlink /etc/systemd/system/ceph.target.wants/ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7.target → /etc/systemd/system/ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7.target. 2023-11-17T05:21:25.126 INFO:teuthology.orchestra.run.smithi040.stderr:Running command: systemctl start ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7.target 2023-11-17T05:21:25.126 INFO:teuthology.orchestra.run.smithi040.stderr:Running command: systemctl daemon-reload 2023-11-17T05:21:25.126 INFO:teuthology.orchestra.run.smithi040.stderr:Running command: systemctl stop ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7@mon.smithi040 2023-11-17T05:21:25.126 INFO:teuthology.orchestra.run.smithi040.stderr:Running command: systemctl reset-failed ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7@mon.smithi040 2023-11-17T05:21:25.182 INFO:teuthology.orchestra.run.smithi040.stderr:systemctl: Failed to reset failed state of unit ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7@mon.smithi040.service: Unit ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7@mon.smithi040.service not loaded. 2023-11-17T05:21:25.182 INFO:teuthology.orchestra.run.smithi040.stderr:Running command: systemctl enable ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7@mon.smithi040 2023-11-17T05:21:25.182 INFO:teuthology.orchestra.run.smithi040.stderr:systemctl: Created symlink /etc/systemd/system/ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7.target.wants/ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7@mon.smithi040.service → /etc/systemd/system/ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7@.service. 2023-11-17T05:21:25.183 INFO:teuthology.orchestra.run.smithi040.stderr:Running command: systemctl start ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7@mon.smithi040 2023-11-17T05:21:25.183 INFO:teuthology.orchestra.run.smithi040.stderr:Running command: systemctl is-enabled firewalld.service 2023-11-17T05:21:25.183 INFO:teuthology.orchestra.run.smithi040.stderr:systemctl: disabled 2023-11-17T05:21:25.183 INFO:teuthology.orchestra.run.smithi040.stderr:Running command: systemctl is-active firewalld.service 2023-11-17T05:21:25.183 INFO:teuthology.orchestra.run.smithi040.stderr:systemctl: inactive 2023-11-17T05:21:25.183 INFO:teuthology.orchestra.run.smithi040.stderr:firewalld.service is not enabled 2023-11-17T05:21:25.183 INFO:teuthology.orchestra.run.smithi040.stderr:Not possible to enable service . firewalld.service is not available 2023-11-17T05:21:25.183 INFO:teuthology.orchestra.run.smithi040.stderr:Not possible to open ports <[]>. firewalld.service is not available 2023-11-17T05:21:25.183 INFO:teuthology.orchestra.run.smithi040.stderr:Waiting for mon to start... 2023-11-17T05:21:25.183 INFO:teuthology.orchestra.run.smithi040.stderr:Waiting for mon... 2023-11-17T05:21:25.183 INFO:teuthology.orchestra.run.smithi040.stderr:Running command: /bin/podman run --rm --ipc=host --no-hosts --net=host --entrypoint /usr/bin/ceph --init -e CONTAINER_IMAGE=docker.io/ceph/ceph:v16.2.4 -e NODE_NAME=smithi040 -e CEPH_USE_RANDOM_NONCE=1 -v /var/lib/ceph/ed6470e2-8508-11ee-8dbe-212e2dc638e7/mon.smithi040:/var/lib/ceph/mon/ceph-smithi040:z -v /tmp/ceph-tmpzx79_7_h:/etc/ceph/ceph.client.admin.keyring:z -v /tmp/ceph-tmpru9tjirw:/etc/ceph/ceph.conf:z docker.io/ceph/ceph:v16.2.4 status 2023-11-17T05:21:25.184 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:15 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111350]: cluster 2023-11-17T05:20:15.837558+0000 mon.smithi040 (mon.0) 1 : cluster [INF] mon.smithi040 is new leader, mons smithi040 in quorum (ranks 0) 2023-11-17T05:21:25.191 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:16.266+0000 7f95eb855700 1 Processor -- start 2023-11-17T05:21:25.191 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:16.266+0000 7f95eb855700 1 -- start start 2023-11-17T05:21:25.191 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:16.267+0000 7f95eb855700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f95e406fc70 0x7f95e4100c00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 rx=0 tx=0).connect 2023-11-17T05:21:25.191 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:16.267+0000 7f95eb855700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f95e40fc680 con 0x7f95e406fc70 2023-11-17T05:21:25.191 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:16.267+0000 7f95e95f1700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f95e406fc70 0x7f95e4100c00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:21:25.192 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:16.267+0000 7f95e95f1700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f95e406fc70 0x7f95e4100c00 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:35660/0 (socket says 172.21.15.40:35660) 2023-11-17T05:21:25.192 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:16.267+0000 7f95e95f1700 1 -- 172.21.15.40:0/2117970641 learned_addr learned my addr 172.21.15.40:0/2117970641 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-11-17T05:21:25.192 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:16.268+0000 7f95e95f1700 1 -- 172.21.15.40:0/2117970641 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f95e41010f0 con 0x7f95e406fc70 2023-11-17T05:21:25.192 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:16.268+0000 7f95e95f1700 1 --2- 172.21.15.40:0/2117970641 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f95e406fc70 0x7f95e4100c00 secure :-1 s=READY pgs=1 cs=0 l=1 rev1=1 rx=0x7f95e000b5f0 tx=0x7f95e0006ee0).ready entity=mon.0 client_cookie=721a9b8499858f server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:21:25.192 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:16.268+0000 7f95d7fff700 1 -- 172.21.15.40:0/2117970641 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f95e000d600 con 0x7f95e406fc70 2023-11-17T05:21:25.192 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:16.268+0000 7f95d7fff700 1 -- 172.21.15.40:0/2117970641 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f95e000d760 con 0x7f95e406fc70 2023-11-17T05:21:25.192 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:16.268+0000 7f95d7fff700 1 -- 172.21.15.40:0/2117970641 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f95e000da30 con 0x7f95e406fc70 2023-11-17T05:21:25.192 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:16.269+0000 7f95eb855700 1 -- 172.21.15.40:0/2117970641 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f95e406fc70 msgr2=0x7f95e4100c00 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:21:25.192 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:16.269+0000 7f95eb855700 1 --2- 172.21.15.40:0/2117970641 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f95e406fc70 0x7f95e4100c00 secure :-1 s=READY pgs=1 cs=0 l=1 rev1=1 rx=0x7f95e000b5f0 tx=0x7f95e0006ee0).stop 2023-11-17T05:21:25.192 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:16.269+0000 7f95eb855700 1 -- 172.21.15.40:0/2117970641 shutdown_connections 2023-11-17T05:21:25.192 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:16.269+0000 7f95eb855700 1 --2- 172.21.15.40:0/2117970641 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f95e406fc70 0x7f95e4100c00 unknown :-1 s=CLOSED pgs=1 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:21:25.192 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:16.269+0000 7f95eb855700 1 -- 172.21.15.40:0/2117970641 >> 172.21.15.40:0/2117970641 conn(0x7f95e40f6280 msgr2=0x7f95e40f86a0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:21:25.192 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:16.269+0000 7f95eb855700 1 -- 172.21.15.40:0/2117970641 shutdown_connections 2023-11-17T05:21:25.192 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:16.269+0000 7f95eb855700 1 -- 172.21.15.40:0/2117970641 wait complete. 2023-11-17T05:21:25.193 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:16.270+0000 7f95eb855700 1 Processor -- start 2023-11-17T05:21:25.193 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:16.270+0000 7f95eb855700 1 -- start start 2023-11-17T05:21:25.193 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:16.271+0000 7f95eb855700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f95e406fc70 0x7f95e410c3d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:21:25.193 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:16.271+0000 7f95eb855700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f95e410c8c0 con 0x7f95e406fc70 2023-11-17T05:21:25.193 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:16.271+0000 7f95e95f1700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f95e406fc70 0x7f95e410c3d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:21:25.193 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:16.271+0000 7f95e95f1700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f95e406fc70 0x7f95e410c3d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:35668/0 (socket says 172.21.15.40:35668) 2023-11-17T05:21:25.193 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:16.271+0000 7f95e95f1700 1 -- 172.21.15.40:0/999034347 learned_addr learned my addr 172.21.15.40:0/999034347 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-11-17T05:21:25.193 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:16.271+0000 7f95e95f1700 1 -- 172.21.15.40:0/999034347 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f95e000b200 con 0x7f95e406fc70 2023-11-17T05:21:25.193 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:16.271+0000 7f95e95f1700 1 --2- 172.21.15.40:0/999034347 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f95e406fc70 0x7f95e410c3d0 secure :-1 s=READY pgs=2 cs=0 l=1 rev1=1 rx=0x7f95e0006c70 tx=0x7f95e0016000).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:21:25.193 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:16.271+0000 7f95d5ffb700 1 -- 172.21.15.40:0/999034347 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f95e0011b40 con 0x7f95e406fc70 2023-11-17T05:21:25.193 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:16.271+0000 7f95d5ffb700 1 -- 172.21.15.40:0/999034347 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f95e0011ca0 con 0x7f95e406fc70 2023-11-17T05:21:25.193 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:16.271+0000 7f95eb855700 1 -- 172.21.15.40:0/999034347 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f95e410cac0 con 0x7f95e406fc70 2023-11-17T05:21:25.193 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:16.271+0000 7f95d5ffb700 1 -- 172.21.15.40:0/999034347 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f95e001d430 con 0x7f95e406fc70 2023-11-17T05:21:25.193 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:16.272+0000 7f95eb855700 1 -- 172.21.15.40:0/999034347 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f95e410ced0 con 0x7f95e406fc70 2023-11-17T05:21:25.194 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:16.272+0000 7f95d5ffb700 1 -- 172.21.15.40:0/999034347 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 1) v1 ==== 484+0+0 (secure 0 0 0) 0x7f95e000f070 con 0x7f95e406fc70 2023-11-17T05:21:25.194 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:16.272+0000 7f95d5ffb700 1 -- 172.21.15.40:0/999034347 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(1..1 src has 1..1) v4 ==== 754+0+0 (secure 0 0 0) 0x7f95e0027460 con 0x7f95e406fc70 2023-11-17T05:21:25.194 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:16.273+0000 7f95eb855700 1 -- 172.21.15.40:0/999034347 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f95e4045ad0 con 0x7f95e406fc70 2023-11-17T05:21:25.194 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:16.275+0000 7f95d5ffb700 1 -- 172.21.15.40:0/999034347 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+72097 (secure 0 0 0) 0x7f95e0016070 con 0x7f95e406fc70 2023-11-17T05:21:25.194 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:16.349+0000 7f95eb855700 1 -- 172.21.15.40:0/999034347 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "status"} v 0) v1 -- 0x7f95e4105fc0 con 0x7f95e406fc70 2023-11-17T05:21:25.194 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:16.349+0000 7f95d5ffb700 1 -- 172.21.15.40:0/999034347 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mon_command_ack([{"prefix": "status"}]=0 v0) v1 ==== 54+0+325 (secure 0 0 0) 0x7f95e001d680 con 0x7f95e406fc70 2023-11-17T05:21:25.218 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: cluster: 2023-11-17T05:21:25.218 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: id: ed6470e2-8508-11ee-8dbe-212e2dc638e7 2023-11-17T05:21:25.218 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: health: HEALTH_OK 2023-11-17T05:21:25.218 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:25.218 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: services: 2023-11-17T05:21:25.218 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: mon: 1 daemons, quorum smithi040 (age 0.496834s) 2023-11-17T05:21:25.218 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: mgr: no daemons active 2023-11-17T05:21:25.218 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: osd: 0 osds: 0 up, 0 in 2023-11-17T05:21:25.218 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:25.219 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: data: 2023-11-17T05:21:25.219 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: pools: 0 pools, 0 pgs 2023-11-17T05:21:25.219 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: objects: 0 objects, 0 B 2023-11-17T05:21:25.219 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: usage: 0 B used, 0 B / 0 B avail 2023-11-17T05:21:25.219 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: pgs: 2023-11-17T05:21:25.219 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:25.219 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:16.350+0000 7f95eb855700 1 -- 172.21.15.40:0/999034347 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f95e406fc70 msgr2=0x7f95e410c3d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:21:25.219 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:16.350+0000 7f95eb855700 1 --2- 172.21.15.40:0/999034347 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f95e406fc70 0x7f95e410c3d0 secure :-1 s=READY pgs=2 cs=0 l=1 rev1=1 rx=0x7f95e0006c70 tx=0x7f95e0016000).stop 2023-11-17T05:21:25.219 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:16.350+0000 7f95eb855700 1 -- 172.21.15.40:0/999034347 shutdown_connections 2023-11-17T05:21:25.219 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:16.350+0000 7f95eb855700 1 --2- 172.21.15.40:0/999034347 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f95e406fc70 0x7f95e410c3d0 unknown :-1 s=CLOSED pgs=2 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:21:25.219 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:16.350+0000 7f95eb855700 1 -- 172.21.15.40:0/999034347 >> 172.21.15.40:0/999034347 conn(0x7f95e40f6280 msgr2=0x7f95e40f8000 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:21:25.219 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:16.350+0000 7f95eb855700 1 -- 172.21.15.40:0/999034347 shutdown_connections 2023-11-17T05:21:25.220 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:16.350+0000 7f95eb855700 1 -- 172.21.15.40:0/999034347 wait complete. 2023-11-17T05:21:25.220 INFO:teuthology.orchestra.run.smithi040.stderr:mon is available 2023-11-17T05:21:25.220 INFO:teuthology.orchestra.run.smithi040.stderr:Assimilating anything we can from ceph.conf... 2023-11-17T05:21:25.220 INFO:teuthology.orchestra.run.smithi040.stderr:Running command: /bin/podman run --rm --ipc=host --no-hosts --net=host --entrypoint /usr/bin/ceph --init -e CONTAINER_IMAGE=docker.io/ceph/ceph:v16.2.4 -e NODE_NAME=smithi040 -e CEPH_USE_RANDOM_NONCE=1 -v /var/log/ceph/ed6470e2-8508-11ee-8dbe-212e2dc638e7:/var/log/ceph:z -v /tmp/ceph-tmpzx79_7_h:/etc/ceph/ceph.client.admin.keyring:z -v /tmp/ceph-tmpru9tjirw:/etc/ceph/ceph.conf:z -v /var/lib/ceph/ed6470e2-8508-11ee-8dbe-212e2dc638e7/mon.smithi040:/var/lib/ceph/mon/ceph-smithi040:z docker.io/ceph/ceph:v16.2.4 config assimilate-conf -i /var/lib/ceph/mon/ceph-smithi040/config 2023-11-17T05:21:25.220 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:17.145+0000 7f75e10d0700 1 Processor -- start 2023-11-17T05:21:25.220 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:17.145+0000 7f75e10d0700 1 -- start start 2023-11-17T05:21:25.220 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:17.145+0000 7f75e10d0700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f75dc06fc70 0x7f75dc100c00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 rx=0 tx=0).connect 2023-11-17T05:21:25.220 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:17.145+0000 7f75e10d0700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f75dc0fc680 con 0x7f75dc06fc70 2023-11-17T05:21:25.220 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:17.145+0000 7f75dad9d700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f75dc06fc70 0x7f75dc100c00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:21:25.220 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:17.145+0000 7f75dad9d700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f75dc06fc70 0x7f75dc100c00 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:35672/0 (socket says 172.21.15.40:35672) 2023-11-17T05:21:25.220 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:17.145+0000 7f75dad9d700 1 -- 172.21.15.40:0/388497691 learned_addr learned my addr 172.21.15.40:0/388497691 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-11-17T05:21:25.220 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:17.146+0000 7f75dad9d700 1 -- 172.21.15.40:0/388497691 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f75dc1010f0 con 0x7f75dc06fc70 2023-11-17T05:21:25.220 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:17.146+0000 7f75dad9d700 1 --2- 172.21.15.40:0/388497691 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f75dc06fc70 0x7f75dc100c00 secure :-1 s=READY pgs=3 cs=0 l=1 rev1=1 rx=0x7f75cc00b5f0 tx=0x7f75cc006ee0).ready entity=mon.0 client_cookie=5be04140d03884a1 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:21:25.220 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:17.146+0000 7f75d9d9b700 1 -- 172.21.15.40:0/388497691 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f75cc00d600 con 0x7f75dc06fc70 2023-11-17T05:21:25.221 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:17.146+0000 7f75d9d9b700 1 -- 172.21.15.40:0/388497691 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f75cc00d760 con 0x7f75dc06fc70 2023-11-17T05:21:25.221 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:17.146+0000 7f75d9d9b700 1 -- 172.21.15.40:0/388497691 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f75cc00da30 con 0x7f75dc06fc70 2023-11-17T05:21:25.221 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:17.146+0000 7f75e10d0700 1 -- 172.21.15.40:0/388497691 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f75dc06fc70 msgr2=0x7f75dc100c00 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:21:25.221 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:17.146+0000 7f75e10d0700 1 --2- 172.21.15.40:0/388497691 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f75dc06fc70 0x7f75dc100c00 secure :-1 s=READY pgs=3 cs=0 l=1 rev1=1 rx=0x7f75cc00b5f0 tx=0x7f75cc006ee0).stop 2023-11-17T05:21:25.221 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:17.146+0000 7f75e10d0700 1 -- 172.21.15.40:0/388497691 shutdown_connections 2023-11-17T05:21:25.221 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:17.146+0000 7f75e10d0700 1 --2- 172.21.15.40:0/388497691 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f75dc06fc70 0x7f75dc100c00 unknown :-1 s=CLOSED pgs=3 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:21:25.221 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:17.146+0000 7f75e10d0700 1 -- 172.21.15.40:0/388497691 >> 172.21.15.40:0/388497691 conn(0x7f75dc0f6280 msgr2=0x7f75dc0f86a0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:21:25.221 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:17.147+0000 7f75e10d0700 1 -- 172.21.15.40:0/388497691 shutdown_connections 2023-11-17T05:21:25.221 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:17.147+0000 7f75e10d0700 1 -- 172.21.15.40:0/388497691 wait complete. 2023-11-17T05:21:25.221 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:17.147+0000 7f75e10d0700 1 Processor -- start 2023-11-17T05:21:25.221 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:17.147+0000 7f75e10d0700 1 -- start start 2023-11-17T05:21:25.221 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:17.147+0000 7f75e10d0700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f75dc06fc70 0x7f75dc10c3d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:21:25.221 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:17.147+0000 7f75e10d0700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f75dc10c8c0 con 0x7f75dc06fc70 2023-11-17T05:21:25.222 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:17.148+0000 7f75dad9d700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f75dc06fc70 0x7f75dc10c3d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:21:25.222 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:17.148+0000 7f75dad9d700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f75dc06fc70 0x7f75dc10c3d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:35674/0 (socket says 172.21.15.40:35674) 2023-11-17T05:21:25.222 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:17.148+0000 7f75dad9d700 1 -- 172.21.15.40:0/3058077366 learned_addr learned my addr 172.21.15.40:0/3058077366 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-11-17T05:21:25.222 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:17.148+0000 7f75dad9d700 1 -- 172.21.15.40:0/3058077366 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f75cc00b200 con 0x7f75dc06fc70 2023-11-17T05:21:25.222 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:17.148+0000 7f75dad9d700 1 --2- 172.21.15.40:0/3058077366 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f75dc06fc70 0x7f75dc10c3d0 secure :-1 s=READY pgs=4 cs=0 l=1 rev1=1 rx=0x7f75cc006c70 tx=0x7f75cc016000).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:21:25.222 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:17.148+0000 7f75c37fe700 1 -- 172.21.15.40:0/3058077366 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f75cc011b40 con 0x7f75dc06fc70 2023-11-17T05:21:25.222 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:17.148+0000 7f75c37fe700 1 -- 172.21.15.40:0/3058077366 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(0 keys) v1 ==== 4+0+0 (secure 0 0 0) 0x7f75cc011ca0 con 0x7f75dc06fc70 2023-11-17T05:21:25.222 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:17.148+0000 7f75e10d0700 1 -- 172.21.15.40:0/3058077366 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f75dc10cac0 con 0x7f75dc06fc70 2023-11-17T05:21:25.249 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:17.148+0000 7f75c37fe700 1 -- 172.21.15.40:0/3058077366 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f75cc01d430 con 0x7f75dc06fc70 2023-11-17T05:21:25.249 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:17.148+0000 7f75e10d0700 1 -- 172.21.15.40:0/3058077366 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f75dc10ced0 con 0x7f75dc06fc70 2023-11-17T05:21:25.249 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:17.149+0000 7f75c37fe700 1 -- 172.21.15.40:0/3058077366 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 1) v1 ==== 484+0+0 (secure 0 0 0) 0x7f75cc00f070 con 0x7f75dc06fc70 2023-11-17T05:21:25.249 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:17.149+0000 7f75c37fe700 1 -- 172.21.15.40:0/3058077366 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(1..1 src has 1..1) v4 ==== 754+0+0 (secure 0 0 0) 0x7f75cc027460 con 0x7f75dc06fc70 2023-11-17T05:21:25.249 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:17.150+0000 7f75e10d0700 1 -- 172.21.15.40:0/3058077366 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f75dc045ad0 con 0x7f75dc06fc70 2023-11-17T05:21:25.249 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:17.152+0000 7f75c37fe700 1 -- 172.21.15.40:0/3058077366 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+72097 (secure 0 0 0) 0x7f75cc016070 con 0x7f75dc06fc70 2023-11-17T05:21:25.249 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:17.222+0000 7f75e10d0700 1 -- 172.21.15.40:0/3058077366 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "config assimilate-conf"} v 0) v1 -- 0x7f75dc101460 con 0x7f75dc06fc70 2023-11-17T05:21:25.249 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:17.225+0000 7f75c37fe700 1 -- 172.21.15.40:0/3058077366 <== mon.0 v2:172.21.15.40:3300/0 7 ==== config(25 keys) v1 ==== 1010+0+0 (secure 0 0 0) 0x7f75cc01d590 con 0x7f75dc06fc70 2023-11-17T05:21:25.249 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:17.225+0000 7f75c37fe700 1 -- 172.21.15.40:0/3058077366 <== mon.0 v2:172.21.15.40:3300/0 8 ==== mon_command_ack([{"prefix": "config assimilate-conf"}]=0 v2) v1 ==== 70+0+464 (secure 0 0 0) 0x7f75cc0276f0 con 0x7f75dc06fc70 2023-11-17T05:21:25.250 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:25.250 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: [global] 2023-11-17T05:21:25.250 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: fsid = ed6470e2-8508-11ee-8dbe-212e2dc638e7 2023-11-17T05:21:25.250 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: mon_host = [v2:172.21.15.40:3300,v1:172.21.15.40:6789] 2023-11-17T05:21:25.250 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: mon_osd_allow_pg_remap = true 2023-11-17T05:21:25.250 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: mon_osd_allow_primary_affinity = true 2023-11-17T05:21:25.250 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: mon_warn_on_no_sortbitwise = false 2023-11-17T05:21:25.250 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: osd_crush_chooseleaf_type = 0 2023-11-17T05:21:25.250 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:25.251 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: [mgr] 2023-11-17T05:21:25.251 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: mgr/telemetry/nag = false 2023-11-17T05:21:25.251 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:25.251 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: [osd] 2023-11-17T05:21:25.251 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: mon_osd_backfillfull_ratio = 0.85 2023-11-17T05:21:25.251 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: mon_osd_full_ratio = 0.9 2023-11-17T05:21:25.251 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: mon_osd_nearfull_ratio = 0.8 2023-11-17T05:21:25.251 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: osd_map_max_advance = 10 2023-11-17T05:21:25.251 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: osd_memory_target_autotune = true 2023-11-17T05:21:25.251 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: osd_sloppy_crc = true 2023-11-17T05:21:25.251 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:17.226+0000 7f75e10d0700 1 -- 172.21.15.40:0/3058077366 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f75dc06fc70 msgr2=0x7f75dc10c3d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:21:25.251 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:17.226+0000 7f75e10d0700 1 --2- 172.21.15.40:0/3058077366 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f75dc06fc70 0x7f75dc10c3d0 secure :-1 s=READY pgs=4 cs=0 l=1 rev1=1 rx=0x7f75cc006c70 tx=0x7f75cc016000).stop 2023-11-17T05:21:25.251 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:17.226+0000 7f75e10d0700 1 -- 172.21.15.40:0/3058077366 shutdown_connections 2023-11-17T05:21:25.252 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:17.226+0000 7f75e10d0700 1 --2- 172.21.15.40:0/3058077366 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f75dc06fc70 0x7f75dc10c3d0 unknown :-1 s=CLOSED pgs=4 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:21:25.252 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:17.226+0000 7f75e10d0700 1 -- 172.21.15.40:0/3058077366 >> 172.21.15.40:0/3058077366 conn(0x7f75dc0f6280 msgr2=0x7f75dc0f8000 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:21:25.252 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:17.226+0000 7f75e10d0700 1 -- 172.21.15.40:0/3058077366 shutdown_connections 2023-11-17T05:21:25.252 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:17.227+0000 7f75e10d0700 1 -- 172.21.15.40:0/3058077366 wait complete. 2023-11-17T05:21:25.252 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:16 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111350]: cluster 2023-11-17T05:20:15.854182+0000 mon.smithi040 (mon.0) 2 : cluster [INF] mon.smithi040 is new leader, mons smithi040 in quorum (ranks 0) 2023-11-17T05:21:25.252 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:16 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111350]: cluster 2023-11-17T05:20:15.854448+0000 mon.smithi040 (mon.0) 3 : cluster [DBG] monmap e1: 1 mons at {smithi040=[v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0]} 2023-11-17T05:21:25.252 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:16 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111350]: cluster 2023-11-17T05:20:15.857258+0000 mon.smithi040 (mon.0) 4 : cluster [DBG] fsmap 2023-11-17T05:21:25.252 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:16 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111350]: cluster 2023-11-17T05:20:15.860892+0000 mon.smithi040 (mon.0) 5 : cluster [DBG] osdmap e1: 0 total, 0 up, 0 in 2023-11-17T05:21:25.252 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:16 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111350]: cluster 2023-11-17T05:20:15.861538+0000 mon.smithi040 (mon.0) 6 : cluster [DBG] mgrmap e1: no daemons active 2023-11-17T05:21:25.252 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:16 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111350]: audit 2023-11-17T05:20:16.350981+0000 mon.smithi040 (mon.0) 7 : audit [DBG] from='client.? 172.21.15.40:0/999034347' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2023-11-17T05:21:25.256 INFO:teuthology.orchestra.run.smithi040.stderr:Generating new minimal ceph.conf... 2023-11-17T05:21:25.256 INFO:teuthology.orchestra.run.smithi040.stderr:Running command: /bin/podman run --rm --ipc=host --no-hosts --net=host --entrypoint /usr/bin/ceph --init -e CONTAINER_IMAGE=docker.io/ceph/ceph:v16.2.4 -e NODE_NAME=smithi040 -e CEPH_USE_RANDOM_NONCE=1 -v /var/log/ceph/ed6470e2-8508-11ee-8dbe-212e2dc638e7:/var/log/ceph:z -v /tmp/ceph-tmpzx79_7_h:/etc/ceph/ceph.client.admin.keyring:z -v /tmp/ceph-tmpru9tjirw:/etc/ceph/ceph.conf:z -v /var/lib/ceph/ed6470e2-8508-11ee-8dbe-212e2dc638e7/mon.smithi040:/var/lib/ceph/mon/ceph-smithi040:z docker.io/ceph/ceph:v16.2.4 config generate-minimal-conf -o /var/lib/ceph/mon/ceph-smithi040/config 2023-11-17T05:21:25.256 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:18.039+0000 7f6dc93c6700 1 Processor -- start 2023-11-17T05:21:25.256 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:18.039+0000 7f6dc93c6700 1 -- start start 2023-11-17T05:21:25.256 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:18.039+0000 7f6dc93c6700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f6dc40fc210 0x7f6dc40fc630 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 rx=0 tx=0).connect 2023-11-17T05:21:25.256 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:18.039+0000 7f6dc93c6700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f6dc40fcb20 con 0x7f6dc40fc210 2023-11-17T05:21:25.257 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:18.039+0000 7f6dc2ffd700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f6dc40fc210 0x7f6dc40fc630 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:21:25.257 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:18.039+0000 7f6dc2ffd700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f6dc40fc210 0x7f6dc40fc630 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:35676/0 (socket says 172.21.15.40:35676) 2023-11-17T05:21:25.257 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:18.039+0000 7f6dc2ffd700 1 -- 172.21.15.40:0/2144428788 learned_addr learned my addr 172.21.15.40:0/2144428788 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-11-17T05:21:25.257 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:18.040+0000 7f6dc2ffd700 1 -- 172.21.15.40:0/2144428788 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f6dc40fcc60 con 0x7f6dc40fc210 2023-11-17T05:21:25.257 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:18.040+0000 7f6dc2ffd700 1 --2- 172.21.15.40:0/2144428788 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f6dc40fc210 0x7f6dc40fc630 secure :-1 s=READY pgs=5 cs=0 l=1 rev1=1 rx=0x7f6db400b5f0 tx=0x7f6db4006ee0).ready entity=mon.0 client_cookie=6a65c5c755a1b5e7 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:21:25.257 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:18.040+0000 7f6dc1ffb700 1 -- 172.21.15.40:0/2144428788 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f6db400d600 con 0x7f6dc40fc210 2023-11-17T05:21:25.257 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:18.040+0000 7f6dc1ffb700 1 -- 172.21.15.40:0/2144428788 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(25 keys) v1 ==== 1010+0+0 (secure 0 0 0) 0x7f6db400dbd0 con 0x7f6dc40fc210 2023-11-17T05:21:25.257 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:18.040+0000 7f6dc1ffb700 1 -- 172.21.15.40:0/2144428788 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f6db4011c00 con 0x7f6dc40fc210 2023-11-17T05:21:25.257 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:18.041+0000 7f6dc93c6700 1 -- 172.21.15.40:0/2144428788 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f6dc40fc210 msgr2=0x7f6dc40fc630 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:21:25.257 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:18.041+0000 7f6dc93c6700 1 --2- 172.21.15.40:0/2144428788 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f6dc40fc210 0x7f6dc40fc630 secure :-1 s=READY pgs=5 cs=0 l=1 rev1=1 rx=0x7f6db400b5f0 tx=0x7f6db4006ee0).stop 2023-11-17T05:21:25.257 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:18.041+0000 7f6dc93c6700 1 -- 172.21.15.40:0/2144428788 shutdown_connections 2023-11-17T05:21:25.257 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:18.041+0000 7f6dc93c6700 1 --2- 172.21.15.40:0/2144428788 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f6dc40fc210 0x7f6dc40fc630 unknown :-1 s=CLOSED pgs=5 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:21:25.257 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:18.041+0000 7f6dc93c6700 1 -- 172.21.15.40:0/2144428788 >> 172.21.15.40:0/2144428788 conn(0x7f6dc40f5a80 msgr2=0x7f6dc40f7ea0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:21:25.258 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:18.041+0000 7f6dc93c6700 1 -- 172.21.15.40:0/2144428788 shutdown_connections 2023-11-17T05:21:25.258 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:18.041+0000 7f6dc93c6700 1 -- 172.21.15.40:0/2144428788 wait complete. 2023-11-17T05:21:25.258 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:18.042+0000 7f6dc93c6700 1 Processor -- start 2023-11-17T05:21:25.258 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:18.042+0000 7f6dc93c6700 1 -- start start 2023-11-17T05:21:25.258 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:18.042+0000 7f6dc93c6700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f6dc40fc210 0x7f6dc410cba0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:21:25.258 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:18.042+0000 7f6dc93c6700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f6dc410d090 con 0x7f6dc40fc210 2023-11-17T05:21:25.258 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:18.042+0000 7f6dc2ffd700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f6dc40fc210 0x7f6dc410cba0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:21:25.258 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:18.043+0000 7f6dc2ffd700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f6dc40fc210 0x7f6dc410cba0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:35690/0 (socket says 172.21.15.40:35690) 2023-11-17T05:21:25.258 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:18.043+0000 7f6dc2ffd700 1 -- 172.21.15.40:0/1894539977 learned_addr learned my addr 172.21.15.40:0/1894539977 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-11-17T05:21:25.258 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:18.043+0000 7f6dc2ffd700 1 -- 172.21.15.40:0/1894539977 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f6db400b200 con 0x7f6dc40fc210 2023-11-17T05:21:25.258 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:18.043+0000 7f6dc2ffd700 1 --2- 172.21.15.40:0/1894539977 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f6dc40fc210 0x7f6dc410cba0 secure :-1 s=READY pgs=6 cs=0 l=1 rev1=1 rx=0x7f6db400e040 tx=0x7f6db4006ee0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:21:25.258 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:18.043+0000 7f6dbb7fe700 1 -- 172.21.15.40:0/1894539977 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f6db401b410 con 0x7f6dc40fc210 2023-11-17T05:21:25.258 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:18.043+0000 7f6dbb7fe700 1 -- 172.21.15.40:0/1894539977 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(25 keys) v1 ==== 1010+0+0 (secure 0 0 0) 0x7f6db401b9e0 con 0x7f6dc40fc210 2023-11-17T05:21:25.259 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:18.043+0000 7f6dbb7fe700 1 -- 172.21.15.40:0/1894539977 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f6db401a770 con 0x7f6dc40fc210 2023-11-17T05:21:25.259 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:18.043+0000 7f6dc93c6700 1 -- 172.21.15.40:0/1894539977 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f6dc410d290 con 0x7f6dc40fc210 2023-11-17T05:21:25.259 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:18.043+0000 7f6dc93c6700 1 -- 172.21.15.40:0/1894539977 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f6dc410d6a0 con 0x7f6dc40fc210 2023-11-17T05:21:25.259 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:18.043+0000 7f6dbb7fe700 1 -- 172.21.15.40:0/1894539977 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 1) v1 ==== 484+0+0 (secure 0 0 0) 0x7f6db400f070 con 0x7f6dc40fc210 2023-11-17T05:21:25.259 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:18.044+0000 7f6dbb7fe700 1 -- 172.21.15.40:0/1894539977 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(1..1 src has 1..1) v4 ==== 754+0+0 (secure 0 0 0) 0x7f6db4024b50 con 0x7f6dc40fc210 2023-11-17T05:21:25.259 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:18.045+0000 7f6dc93c6700 1 -- 172.21.15.40:0/1894539977 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f6dc4045ad0 con 0x7f6dc40fc210 2023-11-17T05:21:25.259 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:18.047+0000 7f6dbb7fe700 1 -- 172.21.15.40:0/1894539977 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+72097 (secure 0 0 0) 0x7f6db4016070 con 0x7f6dc40fc210 2023-11-17T05:21:25.259 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:18.117+0000 7f6dc93c6700 1 -- 172.21.15.40:0/1894539977 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "config generate-minimal-conf"} v 0) v1 -- 0x7f6dc4106a20 con 0x7f6dc40fc210 2023-11-17T05:21:25.259 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:18.118+0000 7f6dbb7fe700 1 -- 172.21.15.40:0/1894539977 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mon_command_ack([{"prefix": "config generate-minimal-conf"}]=0 v2) v1 ==== 76+0+175 (secure 0 0 0) 0x7f6db4024de0 con 0x7f6dc40fc210 2023-11-17T05:21:25.259 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:18.118+0000 7f6dc93c6700 1 -- 172.21.15.40:0/1894539977 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f6dc40fc210 msgr2=0x7f6dc410cba0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:21:25.259 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:18.118+0000 7f6dc93c6700 1 --2- 172.21.15.40:0/1894539977 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f6dc40fc210 0x7f6dc410cba0 secure :-1 s=READY pgs=6 cs=0 l=1 rev1=1 rx=0x7f6db400e040 tx=0x7f6db4006ee0).stop 2023-11-17T05:21:25.259 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:18.118+0000 7f6dc93c6700 1 -- 172.21.15.40:0/1894539977 shutdown_connections 2023-11-17T05:21:25.259 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:18.118+0000 7f6dc93c6700 1 --2- 172.21.15.40:0/1894539977 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f6dc40fc210 0x7f6dc410cba0 unknown :-1 s=CLOSED pgs=6 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:21:25.260 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:18.118+0000 7f6dc93c6700 1 -- 172.21.15.40:0/1894539977 >> 172.21.15.40:0/1894539977 conn(0x7f6dc40f5a80 msgr2=0x7f6dc40f7640 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:21:25.260 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:18.119+0000 7f6dc93c6700 1 -- 172.21.15.40:0/1894539977 shutdown_connections 2023-11-17T05:21:25.260 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:18.119+0000 7f6dc93c6700 1 -- 172.21.15.40:0/1894539977 wait complete. 2023-11-17T05:21:25.260 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:17 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111350]: audit 2023-11-17T05:20:17.224248+0000 mon.smithi040 (mon.0) 8 : audit [INF] from='client.? 172.21.15.40:0/3058077366' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2023-11-17T05:21:25.260 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:17 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111350]: audit 2023-11-17T05:20:17.227447+0000 mon.smithi040 (mon.0) 9 : audit [INF] from='client.? 172.21.15.40:0/3058077366' entity='client.admin' cmd='[{"prefix": "config assimilate-conf"}]': finished 2023-11-17T05:21:25.260 INFO:teuthology.orchestra.run.smithi040.stderr:Restarting the monitor... 2023-11-17T05:21:25.261 INFO:teuthology.orchestra.run.smithi040.stderr:Running command: systemctl restart ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7@mon.smithi040 2023-11-17T05:21:25.261 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:18 smithi040 systemd[1]: Stopping Ceph mon.smithi040 for ed6470e2-8508-11ee-8dbe-212e2dc638e7... 2023-11-17T05:21:25.261 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:18 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111350]: debug 2023-11-17T05:20:18.598+0000 7fc19165c700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.smithi040 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-stderr=true (PID: 1) UID: 0 2023-11-17T05:21:25.261 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:18 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111350]: debug 2023-11-17T05:20:18.598+0000 7fc19165c700 -1 mon.smithi040@0(leader) e1 *** Got Signal Terminated *** 2023-11-17T05:21:25.261 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:18 smithi040 podman[111672]: ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040 2023-11-17T05:21:25.261 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 systemd[1]: ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7@mon.smithi040.service: Succeeded. 2023-11-17T05:21:25.261 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 systemd[1]: Stopped Ceph mon.smithi040 for ed6470e2-8508-11ee-8dbe-212e2dc638e7. 2023-11-17T05:21:25.261 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 systemd[1]: Starting Ceph mon.smithi040 for ed6470e2-8508-11ee-8dbe-212e2dc638e7... 2023-11-17T05:21:25.262 INFO:teuthology.orchestra.run.smithi040.stderr:Setting mon public_network to 172.21.0.0/20 2023-11-17T05:21:25.262 INFO:teuthology.orchestra.run.smithi040.stderr:Running command: /bin/podman run --rm --ipc=host --no-hosts --net=host --entrypoint /usr/bin/ceph --init -e CONTAINER_IMAGE=docker.io/ceph/ceph:v16.2.4 -e NODE_NAME=smithi040 -e CEPH_USE_RANDOM_NONCE=1 -v /var/log/ceph/ed6470e2-8508-11ee-8dbe-212e2dc638e7:/var/log/ceph:z -v /tmp/ceph-tmpzx79_7_h:/etc/ceph/ceph.client.admin.keyring:z -v /tmp/ceph-tmpru9tjirw:/etc/ceph/ceph.conf:z docker.io/ceph/ceph:v16.2.4 config set mon public_network 172.21.0.0/20 2023-11-17T05:21:25.264 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 bash[111744]: 60152325156e54ed990ed0a647f1a24cd92e9a6e733486dcf1c8015ebc6c493e 2023-11-17T05:21:25.264 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: debug 2023-11-17T05:20:19.365+0000 7fab9ff63700 0 set uid:gid to 167:167 (ceph:ceph) 2023-11-17T05:21:25.264 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: debug 2023-11-17T05:20:19.365+0000 7fab9ff63700 0 ceph version 16.2.4 (3cbe25cde3cfa028984618ad32de9edc4c1eaed0) pacific (stable), process ceph-mon, pid 7 2023-11-17T05:21:25.264 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: debug 2023-11-17T05:20:19.365+0000 7fab9ff63700 0 pidfile_write: ignore empty --pid-file 2023-11-17T05:21:25.264 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 systemd[1]: Started Ceph mon.smithi040 for ed6470e2-8508-11ee-8dbe-212e2dc638e7. 2023-11-17T05:21:25.264 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: debug 2023-11-17T05:20:19.370+0000 7fab9ff63700 0 load: jerasure load: lrc load: isa 2023-11-17T05:21:25.264 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: debug 2023-11-17T05:20:19.371+0000 7fab9ff63700 4 rocksdb: RocksDB version: 6.8.1 2023-11-17T05:21:25.264 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: 2023-11-17T05:21:25.264 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: debug 2023-11-17T05:20:19.371+0000 7fab9ff63700 4 rocksdb: Git sha rocksdb_build_git_sha:@0@ 2023-11-17T05:21:25.264 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: debug 2023-11-17T05:20:19.371+0000 7fab9ff63700 4 rocksdb: Compile date May 13 2021 2023-11-17T05:21:25.265 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: debug 2023-11-17T05:20:19.371+0000 7fab9ff63700 4 rocksdb: DB SUMMARY 2023-11-17T05:21:25.265 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: 2023-11-17T05:21:25.265 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: debug 2023-11-17T05:20:19.371+0000 7fab9ff63700 4 rocksdb: CURRENT file: CURRENT 2023-11-17T05:21:25.265 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: 2023-11-17T05:21:25.265 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: debug 2023-11-17T05:20:19.371+0000 7fab9ff63700 4 rocksdb: IDENTITY file: IDENTITY 2023-11-17T05:21:25.265 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: 2023-11-17T05:21:25.265 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: debug 2023-11-17T05:20:19.371+0000 7fab9ff63700 4 rocksdb: MANIFEST file: MANIFEST-000005 size: 131 Bytes 2023-11-17T05:21:25.265 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: 2023-11-17T05:21:25.265 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: debug 2023-11-17T05:20:19.371+0000 7fab9ff63700 4 rocksdb: SST files in /var/lib/ceph/mon/ceph-smithi040/store.db dir, Total Num: 1, files: 000004.sst 2023-11-17T05:21:25.265 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: 2023-11-17T05:21:25.265 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: debug 2023-11-17T05:20:19.371+0000 7fab9ff63700 4 rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-smithi040/store.db: 000006.log size: 92452 ; 2023-11-17T05:21:25.265 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: 2023-11-17T05:21:25.265 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: debug 2023-11-17T05:20:19.371+0000 7fab9ff63700 4 rocksdb: Options.error_if_exists: 0 2023-11-17T05:21:25.265 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: debug 2023-11-17T05:20:19.371+0000 7fab9ff63700 4 rocksdb: Options.create_if_missing: 0 2023-11-17T05:21:25.266 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: debug 2023-11-17T05:20:19.371+0000 7fab9ff63700 4 rocksdb: Options.paranoid_checks: 1 2023-11-17T05:21:25.266 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: debug 2023-11-17T05:20:19.371+0000 7fab9ff63700 4 rocksdb: Options.env: 0x560406aa81c0 2023-11-17T05:21:25.266 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: debug 2023-11-17T05:20:19.371+0000 7fab9ff63700 4 rocksdb: Options.fs: Posix File System 2023-11-17T05:21:25.266 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: debug 2023-11-17T05:20:19.371+0000 7fab9ff63700 4 rocksdb: Options.info_log: 0x5604079e7960 2023-11-17T05:21:25.266 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: debug 2023-11-17T05:20:19.371+0000 7fab9ff63700 4 rocksdb: Options.max_file_opening_threads: 16 2023-11-17T05:21:25.266 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: debug 2023-11-17T05:20:19.371+0000 7fab9ff63700 4 rocksdb: Options.statistics: (nil) 2023-11-17T05:21:25.266 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: debug 2023-11-17T05:20:19.371+0000 7fab9ff63700 4 rocksdb: Options.use_fsync: 0 2023-11-17T05:21:25.266 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: debug 2023-11-17T05:20:19.371+0000 7fab9ff63700 4 rocksdb: Options.max_log_file_size: 0 2023-11-17T05:21:25.266 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: debug 2023-11-17T05:20:19.371+0000 7fab9ff63700 4 rocksdb: Options.max_manifest_file_size: 1073741824 2023-11-17T05:21:25.266 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: debug 2023-11-17T05:20:19.371+0000 7fab9ff63700 4 rocksdb: Options.log_file_time_to_roll: 0 2023-11-17T05:21:25.266 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: debug 2023-11-17T05:20:19.371+0000 7fab9ff63700 4 rocksdb: Options.keep_log_file_num: 1000 2023-11-17T05:21:25.266 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: debug 2023-11-17T05:20:19.371+0000 7fab9ff63700 4 rocksdb: Options.recycle_log_file_num: 0 2023-11-17T05:21:25.267 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: debug 2023-11-17T05:20:19.371+0000 7fab9ff63700 4 rocksdb: Options.allow_fallocate: 1 2023-11-17T05:21:25.267 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: debug 2023-11-17T05:20:19.371+0000 7fab9ff63700 4 rocksdb: Options.allow_mmap_reads: 0 2023-11-17T05:21:25.267 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: debug 2023-11-17T05:20:19.371+0000 7fab9ff63700 4 rocksdb: Options.allow_mmap_writes: 0 2023-11-17T05:21:25.267 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: debug 2023-11-17T05:20:19.371+0000 7fab9ff63700 4 rocksdb: Options.use_direct_reads: 0 2023-11-17T05:21:25.267 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: debug 2023-11-17T05:20:19.371+0000 7fab9ff63700 4 rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2023-11-17T05:21:25.267 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: debug 2023-11-17T05:20:19.371+0000 7fab9ff63700 4 rocksdb: Options.create_missing_column_families: 0 2023-11-17T05:21:25.267 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: debug 2023-11-17T05:20:19.371+0000 7fab9ff63700 4 rocksdb: Options.db_log_dir: 2023-11-17T05:21:25.267 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: debug 2023-11-17T05:20:19.371+0000 7fab9ff63700 4 rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-smithi040/store.db 2023-11-17T05:21:25.267 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: debug 2023-11-17T05:20:19.371+0000 7fab9ff63700 4 rocksdb: Options.table_cache_numshardbits: 6 2023-11-17T05:21:25.267 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: debug 2023-11-17T05:20:19.371+0000 7fab9ff63700 4 rocksdb: Options.max_subcompactions: 1 2023-11-17T05:21:25.267 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: debug 2023-11-17T05:20:19.371+0000 7fab9ff63700 4 rocksdb: Options.max_background_flushes: -1 2023-11-17T05:21:25.267 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: debug 2023-11-17T05:20:19.371+0000 7fab9ff63700 4 rocksdb: Options.WAL_ttl_seconds: 0 2023-11-17T05:21:25.267 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: debug 2023-11-17T05:20:19.371+0000 7fab9ff63700 4 rocksdb: Options.WAL_size_limit_MB: 0 2023-11-17T05:21:25.268 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: debug 2023-11-17T05:20:19.371+0000 7fab9ff63700 4 rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2023-11-17T05:21:25.268 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: debug 2023-11-17T05:20:19.371+0000 7fab9ff63700 4 rocksdb: Options.manifest_preallocation_size: 4194304 2023-11-17T05:21:25.268 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: debug 2023-11-17T05:20:19.371+0000 7fab9ff63700 4 rocksdb: Options.is_fd_close_on_exec: 1 2023-11-17T05:21:25.268 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: debug 2023-11-17T05:20:19.371+0000 7fab9ff63700 4 rocksdb: Options.advise_random_on_open: 1 2023-11-17T05:21:25.268 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: debug 2023-11-17T05:20:19.371+0000 7fab9ff63700 4 rocksdb: Options.db_write_buffer_size: 0 2023-11-17T05:21:25.268 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: debug 2023-11-17T05:20:19.371+0000 7fab9ff63700 4 rocksdb: Options.write_buffer_manager: 0x56040865c030 2023-11-17T05:21:25.268 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: debug 2023-11-17T05:20:19.371+0000 7fab9ff63700 4 rocksdb: Options.access_hint_on_compaction_start: 1 2023-11-17T05:21:25.268 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: debug 2023-11-17T05:20:19.371+0000 7fab9ff63700 4 rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2023-11-17T05:21:25.268 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: debug 2023-11-17T05:20:19.371+0000 7fab9ff63700 4 rocksdb: Options.random_access_max_buffer_size: 1048576 2023-11-17T05:21:25.268 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: debug 2023-11-17T05:20:19.371+0000 7fab9ff63700 4 rocksdb: Options.use_adaptive_mutex: 0 2023-11-17T05:21:25.268 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: debug 2023-11-17T05:20:19.371+0000 7fab9ff63700 4 rocksdb: Options.rate_limiter: (nil) 2023-11-17T05:21:25.268 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: debug 2023-11-17T05:20:19.371+0000 7fab9ff63700 4 rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2023-11-17T05:21:25.268 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: debug 2023-11-17T05:20:19.371+0000 7fab9ff63700 4 rocksdb: Options.wal_recovery_mode: 2 2023-11-17T05:21:25.269 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: debug 2023-11-17T05:20:19.371+0000 7fab9ff63700 4 rocksdb: Options.enable_thread_tracking: 0 2023-11-17T05:21:25.269 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: debug 2023-11-17T05:20:19.371+0000 7fab9ff63700 4 rocksdb: Options.enable_pipelined_write: 0 2023-11-17T05:21:25.269 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: debug 2023-11-17T05:20:19.371+0000 7fab9ff63700 4 rocksdb: Options.unordered_write: 0 2023-11-17T05:21:25.269 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: debug 2023-11-17T05:20:19.371+0000 7fab9ff63700 4 rocksdb: Options.allow_concurrent_memtable_write: 1 2023-11-17T05:21:25.269 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: debug 2023-11-17T05:20:19.371+0000 7fab9ff63700 4 rocksdb: Options.enable_write_thread_adaptive_yield: 1 2023-11-17T05:21:25.269 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: debug 2023-11-17T05:20:19.371+0000 7fab9ff63700 4 rocksdb: Options.write_thread_max_yield_usec: 100 2023-11-17T05:21:25.269 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: debug 2023-11-17T05:20:19.371+0000 7fab9ff63700 4 rocksdb: Options.write_thread_slow_yield_usec: 3 2023-11-17T05:21:25.269 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: debug 2023-11-17T05:20:19.371+0000 7fab9ff63700 4 rocksdb: Options.row_cache: None 2023-11-17T05:21:25.269 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: debug 2023-11-17T05:20:19.371+0000 7fab9ff63700 4 rocksdb: Options.wal_filter: None 2023-11-17T05:21:25.269 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: debug 2023-11-17T05:20:19.371+0000 7fab9ff63700 4 rocksdb: Options.avoid_flush_during_recovery: 0 2023-11-17T05:21:25.269 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: debug 2023-11-17T05:20:19.371+0000 7fab9ff63700 4 rocksdb: Options.allow_ingest_behind: 0 2023-11-17T05:21:25.269 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: debug 2023-11-17T05:20:19.371+0000 7fab9ff63700 4 rocksdb: Options.preserve_deletes: 0 2023-11-17T05:21:25.269 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: debug 2023-11-17T05:20:19.371+0000 7fab9ff63700 4 rocksdb: Options.two_write_queues: 0 2023-11-17T05:21:25.269 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: debug 2023-11-17T05:20:19.371+0000 7fab9ff63700 4 rocksdb: Options.manual_wal_flush: 0 2023-11-17T05:21:25.270 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: debug 2023-11-17T05:20:19.371+0000 7fab9ff63700 4 rocksdb: Options.atomic_flush: 0 2023-11-17T05:21:25.270 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: debug 2023-11-17T05:20:19.371+0000 7fab9ff63700 4 rocksdb: Options.avoid_unnecessary_blocking_io: 0 2023-11-17T05:21:25.270 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: debug 2023-11-17T05:20:19.371+0000 7fab9ff63700 4 rocksdb: Options.persist_stats_to_disk: 0 2023-11-17T05:21:25.270 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: debug 2023-11-17T05:20:19.371+0000 7fab9ff63700 4 rocksdb: Options.write_dbid_to_manifest: 0 2023-11-17T05:21:25.270 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: debug 2023-11-17T05:20:19.371+0000 7fab9ff63700 4 rocksdb: Options.log_readahead_size: 0 2023-11-17T05:21:25.270 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: debug 2023-11-17T05:20:19.371+0000 7fab9ff63700 4 rocksdb: Options.sst_file_checksum_func: Unknown 2023-11-17T05:21:25.270 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: debug 2023-11-17T05:20:19.371+0000 7fab9ff63700 4 rocksdb: Options.max_background_jobs: 2 2023-11-17T05:21:25.270 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: debug 2023-11-17T05:20:19.371+0000 7fab9ff63700 4 rocksdb: Options.max_background_compactions: -1 2023-11-17T05:21:25.270 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: debug 2023-11-17T05:20:19.371+0000 7fab9ff63700 4 rocksdb: Options.avoid_flush_during_shutdown: 0 2023-11-17T05:21:25.270 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: debug 2023-11-17T05:20:19.371+0000 7fab9ff63700 4 rocksdb: Options.writable_file_max_buffer_size: 1048576 2023-11-17T05:21:25.270 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: debug 2023-11-17T05:20:19.371+0000 7fab9ff63700 4 rocksdb: Options.delayed_write_rate : 16777216 2023-11-17T05:21:25.270 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: debug 2023-11-17T05:20:19.371+0000 7fab9ff63700 4 rocksdb: Options.max_total_wal_size: 0 2023-11-17T05:21:25.270 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: debug 2023-11-17T05:20:19.371+0000 7fab9ff63700 4 rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2023-11-17T05:21:25.271 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: debug 2023-11-17T05:20:19.371+0000 7fab9ff63700 4 rocksdb: Options.stats_dump_period_sec: 600 2023-11-17T05:21:25.271 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: debug 2023-11-17T05:20:19.371+0000 7fab9ff63700 4 rocksdb: Options.stats_persist_period_sec: 600 2023-11-17T05:21:25.271 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: debug 2023-11-17T05:20:19.371+0000 7fab9ff63700 4 rocksdb: Options.stats_history_buffer_size: 1048576 2023-11-17T05:21:25.271 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: debug 2023-11-17T05:20:19.371+0000 7fab9ff63700 4 rocksdb: Options.max_open_files: -1 2023-11-17T05:21:25.271 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: debug 2023-11-17T05:20:19.371+0000 7fab9ff63700 4 rocksdb: Options.bytes_per_sync: 0 2023-11-17T05:21:25.271 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: debug 2023-11-17T05:20:19.371+0000 7fab9ff63700 4 rocksdb: Options.wal_bytes_per_sync: 0 2023-11-17T05:21:25.271 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: debug 2023-11-17T05:20:19.371+0000 7fab9ff63700 4 rocksdb: Options.strict_bytes_per_sync: 0 2023-11-17T05:21:25.271 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: debug 2023-11-17T05:20:19.371+0000 7fab9ff63700 4 rocksdb: Options.compaction_readahead_size: 0 2023-11-17T05:21:25.271 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: debug 2023-11-17T05:20:19.371+0000 7fab9ff63700 4 rocksdb: Compression algorithms supported: 2023-11-17T05:21:25.271 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: debug 2023-11-17T05:20:19.371+0000 7fab9ff63700 4 rocksdb: kZSTDNotFinalCompression supported: 0 2023-11-17T05:21:25.271 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: debug 2023-11-17T05:20:19.371+0000 7fab9ff63700 4 rocksdb: kZSTD supported: 0 2023-11-17T05:21:25.271 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: debug 2023-11-17T05:20:19.371+0000 7fab9ff63700 4 rocksdb: kXpressCompression supported: 0 2023-11-17T05:21:25.311 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: debug 2023-11-17T05:20:19.371+0000 7fab9ff63700 4 rocksdb: kLZ4HCCompression supported: 1 2023-11-17T05:21:25.311 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: debug 2023-11-17T05:20:19.371+0000 7fab9ff63700 4 rocksdb: kLZ4Compression supported: 1 2023-11-17T05:21:25.312 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: debug 2023-11-17T05:20:19.371+0000 7fab9ff63700 4 rocksdb: kBZip2Compression supported: 0 2023-11-17T05:21:25.312 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: debug 2023-11-17T05:20:19.371+0000 7fab9ff63700 4 rocksdb: kZlibCompression supported: 1 2023-11-17T05:21:25.312 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: debug 2023-11-17T05:20:19.371+0000 7fab9ff63700 4 rocksdb: kSnappyCompression supported: 1 2023-11-17T05:21:25.312 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: debug 2023-11-17T05:20:19.371+0000 7fab9ff63700 4 rocksdb: Fast CRC32 supported: Supported on x86 2023-11-17T05:21:25.312 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: debug 2023-11-17T05:20:19.371+0000 7fab9ff63700 4 rocksdb: [version_set.cc:4413] Recovering from manifest file: /var/lib/ceph/mon/ceph-smithi040/store.db/MANIFEST-000005 2023-11-17T05:21:25.312 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: 2023-11-17T05:21:25.312 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: debug 2023-11-17T05:20:19.371+0000 7fab9ff63700 4 rocksdb: [column_family.cc:552] --------------- Options for column family [default]: 2023-11-17T05:21:25.312 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: 2023-11-17T05:21:25.312 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: debug 2023-11-17T05:20:19.371+0000 7fab9ff63700 4 rocksdb: Options.comparator: leveldb.BytewiseComparator 2023-11-17T05:21:25.312 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: debug 2023-11-17T05:20:19.371+0000 7fab9ff63700 4 rocksdb: Options.merge_operator: 2023-11-17T05:21:25.312 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: debug 2023-11-17T05:20:19.371+0000 7fab9ff63700 4 rocksdb: Options.compaction_filter: None 2023-11-17T05:21:25.312 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: debug 2023-11-17T05:20:19.371+0000 7fab9ff63700 4 rocksdb: Options.compaction_filter_factory: None 2023-11-17T05:21:25.312 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: debug 2023-11-17T05:20:19.371+0000 7fab9ff63700 4 rocksdb: Options.memtable_factory: SkipListFactory 2023-11-17T05:21:25.312 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: debug 2023-11-17T05:20:19.371+0000 7fab9ff63700 4 rocksdb: Options.table_factory: BlockBasedTable 2023-11-17T05:21:25.313 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: debug 2023-11-17T05:20:19.371+0000 7fab9ff63700 4 rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x5604078fe128) 2023-11-17T05:21:25.313 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cache_index_and_filter_blocks: 1 2023-11-17T05:21:25.313 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cache_index_and_filter_blocks_with_high_priority: 0 2023-11-17T05:21:25.313 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: pin_l0_filter_and_index_blocks_in_cache: 0 2023-11-17T05:21:25.313 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: pin_top_level_index_and_filter: 1 2023-11-17T05:21:25.313 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: index_type: 0 2023-11-17T05:21:25.313 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: data_block_index_type: 0 2023-11-17T05:21:25.313 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: index_shortening: 1 2023-11-17T05:21:25.313 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: data_block_hash_table_util_ratio: 0.750000 2023-11-17T05:21:25.313 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: hash_index_allow_collision: 1 2023-11-17T05:21:25.313 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: checksum: 1 2023-11-17T05:21:25.313 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: no_block_cache: 0 2023-11-17T05:21:25.313 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: block_cache: 0x560407934f10 2023-11-17T05:21:25.314 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: block_cache_name: BinnedLRUCache 2023-11-17T05:21:25.314 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: block_cache_options: 2023-11-17T05:21:25.314 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: capacity : 536870912 2023-11-17T05:21:25.314 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: num_shard_bits : 4 2023-11-17T05:21:25.314 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: strict_capacity_limit : 0 2023-11-17T05:21:25.314 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: high_pri_pool_ratio: 0.000 2023-11-17T05:21:25.314 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: block_cache_compressed: (nil) 2023-11-17T05:21:25.314 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: persistent_cache: (nil) 2023-11-17T05:21:25.314 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: block_size: 4096 2023-11-17T05:21:25.314 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: block_size_deviation: 10 2023-11-17T05:21:25.314 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: block_restart_interval: 16 2023-11-17T05:21:25.314 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: index_block_restart_interval: 1 2023-11-17T05:21:25.314 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: metadata_block_size: 4096 2023-11-17T05:21:25.315 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: partition_filters: 0 2023-11-17T05:21:25.315 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: use_delta_encoding: 1 2023-11-17T05:21:25.315 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: filter_policy: rocksdb.BuiltinBloomFilter 2023-11-17T05:21:25.315 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: whole_key_filtering: 1 2023-11-17T05:21:25.315 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: verify_compression: 0 2023-11-17T05:21:25.315 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: read_amp_bytes_per_bit: 0 2023-11-17T05:21:25.315 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: format_version: 2 2023-11-17T05:21:25.315 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: enable_index_compression: 1 2023-11-17T05:21:25.315 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: block_align: 0 2023-11-17T05:21:25.315 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: 2023-11-17T05:21:25.315 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: debug 2023-11-17T05:20:19.371+0000 7fab9ff63700 4 rocksdb: Options.write_buffer_size: 33554432 2023-11-17T05:21:25.315 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: debug 2023-11-17T05:20:19.371+0000 7fab9ff63700 4 rocksdb: Options.max_write_buffer_number: 2 2023-11-17T05:21:25.315 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: debug 2023-11-17T05:20:19.371+0000 7fab9ff63700 4 rocksdb: Options.compression: NoCompression 2023-11-17T05:21:25.316 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: debug 2023-11-17T05:20:19.371+0000 7fab9ff63700 4 rocksdb: Options.bottommost_compression: Disabled 2023-11-17T05:21:25.316 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: debug 2023-11-17T05:20:19.371+0000 7fab9ff63700 4 rocksdb: Options.prefix_extractor: nullptr 2023-11-17T05:21:25.316 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: debug 2023-11-17T05:20:19.371+0000 7fab9ff63700 4 rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2023-11-17T05:21:25.316 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: debug 2023-11-17T05:20:19.371+0000 7fab9ff63700 4 rocksdb: Options.num_levels: 7 2023-11-17T05:21:25.358 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: debug 2023-11-17T05:20:19.371+0000 7fab9ff63700 4 rocksdb: Options.min_write_buffer_number_to_merge: 1 2023-11-17T05:21:25.358 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: debug 2023-11-17T05:20:19.371+0000 7fab9ff63700 4 rocksdb: Options.max_write_buffer_number_to_maintain: 0 2023-11-17T05:21:25.359 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: debug 2023-11-17T05:20:19.371+0000 7fab9ff63700 4 rocksdb: Options.max_write_buffer_size_to_maintain: 0 2023-11-17T05:21:25.359 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: debug 2023-11-17T05:20:19.371+0000 7fab9ff63700 4 rocksdb: Options.bottommost_compression_opts.window_bits: -14 2023-11-17T05:21:25.359 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: debug 2023-11-17T05:20:19.371+0000 7fab9ff63700 4 rocksdb: Options.bottommost_compression_opts.level: 32767 2023-11-17T05:21:25.360 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: debug 2023-11-17T05:20:19.371+0000 7fab9ff63700 4 rocksdb: Options.bottommost_compression_opts.strategy: 0 2023-11-17T05:21:25.360 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: debug 2023-11-17T05:20:19.371+0000 7fab9ff63700 4 rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2023-11-17T05:21:25.360 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: debug 2023-11-17T05:20:19.371+0000 7fab9ff63700 4 rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2023-11-17T05:21:25.360 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: debug 2023-11-17T05:20:19.371+0000 7fab9ff63700 4 rocksdb: Options.bottommost_compression_opts.enabled: false 2023-11-17T05:21:25.360 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: debug 2023-11-17T05:20:19.371+0000 7fab9ff63700 4 rocksdb: Options.compression_opts.window_bits: -14 2023-11-17T05:21:25.360 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: debug 2023-11-17T05:20:19.371+0000 7fab9ff63700 4 rocksdb: Options.compression_opts.level: 32767 2023-11-17T05:21:25.360 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: debug 2023-11-17T05:20:19.371+0000 7fab9ff63700 4 rocksdb: Options.compression_opts.strategy: 0 2023-11-17T05:21:25.360 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: debug 2023-11-17T05:20:19.371+0000 7fab9ff63700 4 rocksdb: Options.compression_opts.max_dict_bytes: 0 2023-11-17T05:21:25.360 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: debug 2023-11-17T05:20:19.371+0000 7fab9ff63700 4 rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2023-11-17T05:21:25.360 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: debug 2023-11-17T05:20:19.371+0000 7fab9ff63700 4 rocksdb: Options.compression_opts.enabled: false 2023-11-17T05:21:25.360 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: debug 2023-11-17T05:20:19.371+0000 7fab9ff63700 4 rocksdb: Options.level0_file_num_compaction_trigger: 4 2023-11-17T05:21:25.360 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: debug 2023-11-17T05:20:19.371+0000 7fab9ff63700 4 rocksdb: Options.level0_slowdown_writes_trigger: 20 2023-11-17T05:21:25.360 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: debug 2023-11-17T05:20:19.371+0000 7fab9ff63700 4 rocksdb: Options.level0_stop_writes_trigger: 36 2023-11-17T05:21:25.361 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: debug 2023-11-17T05:20:19.371+0000 7fab9ff63700 4 rocksdb: Options.target_file_size_base: 67108864 2023-11-17T05:21:25.361 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: debug 2023-11-17T05:20:19.371+0000 7fab9ff63700 4 rocksdb: Options.target_file_size_multiplier: 1 2023-11-17T05:21:25.361 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: debug 2023-11-17T05:20:19.371+0000 7fab9ff63700 4 rocksdb: Options.max_bytes_for_level_base: 268435456 2023-11-17T05:21:25.361 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: debug 2023-11-17T05:20:19.371+0000 7fab9ff63700 4 rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2023-11-17T05:21:25.361 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: debug 2023-11-17T05:20:19.371+0000 7fab9ff63700 4 rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2023-11-17T05:21:25.361 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: debug 2023-11-17T05:20:19.371+0000 7fab9ff63700 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2023-11-17T05:21:25.361 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: debug 2023-11-17T05:20:19.371+0000 7fab9ff63700 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2023-11-17T05:21:25.361 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: debug 2023-11-17T05:20:19.371+0000 7fab9ff63700 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2023-11-17T05:21:25.361 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: debug 2023-11-17T05:20:19.371+0000 7fab9ff63700 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2023-11-17T05:21:25.361 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: debug 2023-11-17T05:20:19.371+0000 7fab9ff63700 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2023-11-17T05:21:25.361 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: debug 2023-11-17T05:20:19.371+0000 7fab9ff63700 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2023-11-17T05:21:25.401 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: debug 2023-11-17T05:20:19.371+0000 7fab9ff63700 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2023-11-17T05:21:25.401 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: debug 2023-11-17T05:20:19.371+0000 7fab9ff63700 4 rocksdb: Options.max_sequential_skip_in_iterations: 8 2023-11-17T05:21:25.401 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: debug 2023-11-17T05:20:19.371+0000 7fab9ff63700 4 rocksdb: Options.max_compaction_bytes: 1677721600 2023-11-17T05:21:25.401 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: debug 2023-11-17T05:20:19.371+0000 7fab9ff63700 4 rocksdb: Options.arena_block_size: 4194304 2023-11-17T05:21:25.401 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: debug 2023-11-17T05:20:19.371+0000 7fab9ff63700 4 rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2023-11-17T05:21:25.446 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: debug 2023-11-17T05:20:19.371+0000 7fab9ff63700 4 rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2023-11-17T05:21:25.446 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: debug 2023-11-17T05:20:19.371+0000 7fab9ff63700 4 rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2023-11-17T05:21:25.446 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: debug 2023-11-17T05:20:19.371+0000 7fab9ff63700 4 rocksdb: Options.disable_auto_compactions: 0 2023-11-17T05:21:25.446 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: debug 2023-11-17T05:20:19.371+0000 7fab9ff63700 4 rocksdb: Options.compaction_style: kCompactionStyleLevel 2023-11-17T05:21:25.446 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: debug 2023-11-17T05:20:19.371+0000 7fab9ff63700 4 rocksdb: Options.compaction_pri: kMinOverlappingRatio 2023-11-17T05:21:25.446 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: debug 2023-11-17T05:20:19.371+0000 7fab9ff63700 4 rocksdb: Options.compaction_options_universal.size_ratio: 1 2023-11-17T05:21:25.446 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: debug 2023-11-17T05:20:19.371+0000 7fab9ff63700 4 rocksdb: Options.compaction_options_universal.min_merge_width: 2 2023-11-17T05:21:25.447 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: debug 2023-11-17T05:20:19.371+0000 7fab9ff63700 4 rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2023-11-17T05:21:25.447 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: debug 2023-11-17T05:20:19.371+0000 7fab9ff63700 4 rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2023-11-17T05:21:25.447 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: debug 2023-11-17T05:20:19.371+0000 7fab9ff63700 4 rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2023-11-17T05:21:25.447 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: debug 2023-11-17T05:20:19.371+0000 7fab9ff63700 4 rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2023-11-17T05:21:25.661 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: debug 2023-11-17T05:20:19.371+0000 7fab9ff63700 4 rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2023-11-17T05:21:25.661 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: debug 2023-11-17T05:20:19.371+0000 7fab9ff63700 4 rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2023-11-17T05:21:25.661 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: debug 2023-11-17T05:20:19.371+0000 7fab9ff63700 4 rocksdb: Options.table_properties_collectors: 2023-11-17T05:21:25.661 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: debug 2023-11-17T05:20:19.371+0000 7fab9ff63700 4 rocksdb: Options.inplace_update_support: 0 2023-11-17T05:21:25.661 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: debug 2023-11-17T05:20:19.371+0000 7fab9ff63700 4 rocksdb: Options.inplace_update_num_locks: 10000 2023-11-17T05:21:25.662 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: debug 2023-11-17T05:20:19.371+0000 7fab9ff63700 4 rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2023-11-17T05:21:25.662 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: debug 2023-11-17T05:20:19.371+0000 7fab9ff63700 4 rocksdb: Options.memtable_whole_key_filtering: 0 2023-11-17T05:21:25.662 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: debug 2023-11-17T05:20:19.371+0000 7fab9ff63700 4 rocksdb: Options.memtable_huge_page_size: 0 2023-11-17T05:21:25.662 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: debug 2023-11-17T05:20:19.371+0000 7fab9ff63700 4 rocksdb: Options.bloom_locality: 0 2023-11-17T05:21:25.662 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: debug 2023-11-17T05:20:19.371+0000 7fab9ff63700 4 rocksdb: Options.max_successive_merges: 0 2023-11-17T05:21:25.663 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: debug 2023-11-17T05:20:19.371+0000 7fab9ff63700 4 rocksdb: Options.optimize_filters_for_hits: 0 2023-11-17T05:21:25.663 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: debug 2023-11-17T05:20:19.371+0000 7fab9ff63700 4 rocksdb: Options.paranoid_file_checks: 0 2023-11-17T05:21:25.663 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: debug 2023-11-17T05:20:19.371+0000 7fab9ff63700 4 rocksdb: Options.force_consistency_checks: 0 2023-11-17T05:21:25.663 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: debug 2023-11-17T05:20:19.371+0000 7fab9ff63700 4 rocksdb: Options.report_bg_io_stats: 0 2023-11-17T05:21:25.663 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: debug 2023-11-17T05:20:19.371+0000 7fab9ff63700 4 rocksdb: Options.ttl: 2592000 2023-11-17T05:21:25.663 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: debug 2023-11-17T05:20:19.371+0000 7fab9ff63700 4 rocksdb: Options.periodic_compaction_seconds: 0 2023-11-17T05:21:25.664 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: debug 2023-11-17T05:20:19.372+0000 7fab9ff63700 4 rocksdb: [version_set.cc:4568] Recovered from manifest file:/var/lib/ceph/mon/ceph-smithi040/store.db/MANIFEST-000005 succeeded,manifest_file_number is 5, next_file_number is 7, last_sequence is 5, log_number is 4,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2023-11-17T05:21:25.664 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: 2023-11-17T05:21:25.664 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: debug 2023-11-17T05:20:19.372+0000 7fab9ff63700 4 rocksdb: [version_set.cc:4577] Column family [default] (ID 0), log number is 4 2023-11-17T05:21:25.664 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: 2023-11-17T05:21:25.664 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: debug 2023-11-17T05:20:19.372+0000 7fab9ff63700 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1700198419374372, "job": 1, "event": "recovery_started", "log_files": [6]} 2023-11-17T05:21:25.664 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: debug 2023-11-17T05:20:19.372+0000 7fab9ff63700 4 rocksdb: [db_impl/db_impl_open.cc:760] Recovering log #6 mode 2 2023-11-17T05:21:25.665 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: debug 2023-11-17T05:20:19.373+0000 7fab9ff63700 3 rocksdb: [le/block_based/filter_policy.cc:584] Using legacy Bloom filter with high (20) bits/key. Dramatic filter space and/or accuracy improvement is available with format_version>=5. 2023-11-17T05:21:25.665 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: debug 2023-11-17T05:20:19.373+0000 7fab9ff63700 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1700198419375504, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 7, "file_size": 88252, "table_properties": {"data_size": 86379, "index_size": 350, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 0, "index_value_is_delta_encoded": 0, "filter_size": 709, "raw_key_size": 12269, "raw_average_key_size": 49, "raw_value_size": 79593, "raw_average_value_size": 322, "num_data_blocks": 11, "num_entries": 247, "num_deletions": 3, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "rocksdb.BuiltinBloomFilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; ", "creation_time": 1700198419, "oldest_key_time": 3, "file_creation_time": 0}} 2023-11-17T05:21:25.665 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: debug 2023-11-17T05:20:19.373+0000 7fab9ff63700 4 rocksdb: [version_set.cc:3826] Creating manifest 8 2023-11-17T05:21:25.740 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: 2023-11-17T05:21:25.740 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: debug 2023-11-17T05:20:19.374+0000 7fab9ff63700 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1700198419376031, "job": 1, "event": "recovery_finished"} 2023-11-17T05:21:25.740 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: debug 2023-11-17T05:20:19.375+0000 7fab9ff63700 4 rocksdb: DB pointer 0x5604079f9800 2023-11-17T05:21:25.740 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: debug 2023-11-17T05:20:19.375+0000 7fab86d7b700 4 rocksdb: [db_impl/db_impl.cc:850] ------- DUMPING STATS ------- 2023-11-17T05:21:25.740 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: debug 2023-11-17T05:20:19.375+0000 7fab86d7b700 4 rocksdb: [db_impl/db_impl.cc:851] 2023-11-17T05:21:25.741 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: ** DB Stats ** 2023-11-17T05:21:25.741 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: Uptime(secs): 0.0 total, 0.0 interval 2023-11-17T05:21:25.741 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2023-11-17T05:21:25.741 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2023-11-17T05:21:25.741 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2023-11-17T05:21:25.741 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2023-11-17T05:21:25.741 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2023-11-17T05:21:25.741 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2023-11-17T05:21:25.741 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: 2023-11-17T05:21:25.741 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: ** Compaction Stats [default] ** 2023-11-17T05:21:25.741 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-11-17T05:21:25.741 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-11-17T05:21:25.741 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: L0 2/0 87.88 KB 0.5 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 118.5 0.00 0.00 1 0.001 0 0 2023-11-17T05:21:25.742 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: Sum 2/0 87.88 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 118.5 0.00 0.00 1 0.001 0 0 2023-11-17T05:21:25.742 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 118.5 0.00 0.00 1 0.001 0 0 2023-11-17T05:21:25.742 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: 2023-11-17T05:21:25.742 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: ** Compaction Stats [default] ** 2023-11-17T05:21:25.742 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-11-17T05:21:25.742 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-11-17T05:21:25.742 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 118.5 0.00 0.00 1 0.001 0 0 2023-11-17T05:21:25.742 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: Uptime(secs): 0.0 total, 0.0 interval 2023-11-17T05:21:25.742 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: Flush(GB): cumulative 0.000, interval 0.000 2023-11-17T05:21:25.742 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: AddFile(GB): cumulative 0.000, interval 0.000 2023-11-17T05:21:25.742 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: AddFile(Total Files): cumulative 0, interval 0 2023-11-17T05:21:25.742 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: AddFile(L0 Files): cumulative 0, interval 0 2023-11-17T05:21:25.742 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: AddFile(Keys): cumulative 0, interval 0 2023-11-17T05:21:25.743 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: Cumulative compaction: 0.00 GB write, 22.08 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-11-17T05:21:25.743 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: Interval compaction: 0.00 GB write, 22.08 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-11-17T05:21:25.743 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-11-17T05:21:25.743 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: 2023-11-17T05:21:25.743 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: ** File Read Latency Histogram By Level [default] ** 2023-11-17T05:21:25.743 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: 2023-11-17T05:21:25.743 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: ** Compaction Stats [default] ** 2023-11-17T05:21:25.743 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-11-17T05:21:25.743 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-11-17T05:21:25.743 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: L0 2/0 87.88 KB 0.5 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 118.5 0.00 0.00 1 0.001 0 0 2023-11-17T05:21:25.743 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: Sum 2/0 87.88 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 118.5 0.00 0.00 1 0.001 0 0 2023-11-17T05:21:25.743 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2023-11-17T05:21:25.743 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: 2023-11-17T05:21:25.744 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: ** Compaction Stats [default] ** 2023-11-17T05:21:25.744 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-11-17T05:21:25.744 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-11-17T05:21:25.744 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 118.5 0.00 0.00 1 0.001 0 0 2023-11-17T05:21:25.744 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: Uptime(secs): 0.0 total, 0.0 interval 2023-11-17T05:21:25.744 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: Flush(GB): cumulative 0.000, interval 0.000 2023-11-17T05:21:25.766 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: AddFile(GB): cumulative 0.000, interval 0.000 2023-11-17T05:21:25.766 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: AddFile(Total Files): cumulative 0, interval 0 2023-11-17T05:21:25.766 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: AddFile(L0 Files): cumulative 0, interval 0 2023-11-17T05:21:25.766 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: AddFile(Keys): cumulative 0, interval 0 2023-11-17T05:21:25.766 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: Cumulative compaction: 0.00 GB write, 21.78 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-11-17T05:21:25.766 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-11-17T05:21:25.766 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-11-17T05:21:25.767 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: 2023-11-17T05:21:25.767 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: ** File Read Latency Histogram By Level [default] ** 2023-11-17T05:21:25.767 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: 2023-11-17T05:21:25.767 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: debug 2023-11-17T05:20:19.376+0000 7fab9ff63700 0 starting mon.smithi040 rank 0 at public addrs [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] at bind addrs [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] mon_data /var/lib/ceph/mon/ceph-smithi040 fsid ed6470e2-8508-11ee-8dbe-212e2dc638e7 2023-11-17T05:21:25.767 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: debug 2023-11-17T05:20:19.377+0000 7fab9ff63700 1 mon.smithi040@-1(???) e1 preinit fsid ed6470e2-8508-11ee-8dbe-212e2dc638e7 2023-11-17T05:21:25.767 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: debug 2023-11-17T05:20:19.377+0000 7fab9ff63700 0 mon.smithi040@-1(???).mds e1 new map 2023-11-17T05:21:25.767 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: debug 2023-11-17T05:20:19.377+0000 7fab9ff63700 0 mon.smithi040@-1(???).mds e1 print_map 2023-11-17T05:21:25.767 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: e1 2023-11-17T05:21:25.767 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: enable_multiple, ever_enabled_multiple: 1,1 2023-11-17T05:21:25.767 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2023-11-17T05:21:25.768 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: legacy client fscid: -1 2023-11-17T05:21:25.768 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: 2023-11-17T05:21:25.770 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: No filesystems configured 2023-11-17T05:21:25.770 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: 2023-11-17T05:21:25.771 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: debug 2023-11-17T05:20:19.377+0000 7fab9ff63700 0 mon.smithi040@-1(???).osd e1 crush map has features 3314932999778484224, adjusting msgr requires 2023-11-17T05:21:25.771 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: debug 2023-11-17T05:20:19.377+0000 7fab9ff63700 0 mon.smithi040@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2023-11-17T05:21:25.771 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: debug 2023-11-17T05:20:19.377+0000 7fab9ff63700 0 mon.smithi040@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2023-11-17T05:21:25.800 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: debug 2023-11-17T05:20:19.377+0000 7fab9ff63700 0 mon.smithi040@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2023-11-17T05:21:25.800 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:20:17.224248+0000 mon.smithi040 (mon.0) 8 : audit [INF] from='client.? 172.21.15.40:0/3058077366' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2023-11-17T05:21:25.800 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:20:17.227447+0000 mon.smithi040 (mon.0) 9 : audit [INF] from='client.? 172.21.15.40:0/3058077366' entity='client.admin' cmd='[{"prefix": "config assimilate-conf"}]': finished 2023-11-17T05:21:25.802 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: debug 2023-11-17T05:20:19.377+0000 7fab9ff63700 1 mon.smithi040@-1(???).paxosservice(auth 1..2) refresh upgraded, format 0 -> 3 2023-11-17T05:21:25.803 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:20:19.396454+0000 mon.smithi040 (mon.0) 1 : cluster [INF] mon.smithi040 is new leader, mons smithi040 in quorum (ranks 0) 2023-11-17T05:21:25.804 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:20:19.396527+0000 mon.smithi040 (mon.0) 2 : cluster [DBG] monmap e1: 1 mons at {smithi040=[v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0]} 2023-11-17T05:21:25.804 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:20:19.396586+0000 mon.smithi040 (mon.0) 3 : cluster [DBG] fsmap 2023-11-17T05:21:25.804 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:20:19.396618+0000 mon.smithi040 (mon.0) 4 : cluster [DBG] osdmap e1: 0 total, 0 up, 0 in 2023-11-17T05:21:25.804 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:20:19.397100+0000 mon.smithi040 (mon.0) 5 : cluster [DBG] mgrmap e1: no daemons active 2023-11-17T05:21:25.812 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:19.763+0000 7fc17c5c7700 1 Processor -- start 2023-11-17T05:21:25.812 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:19.763+0000 7fc17c5c7700 1 -- start start 2023-11-17T05:21:25.813 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:19.764+0000 7fc17c5c7700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fc1740faae0 0x7fc1740faf00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 rx=0 tx=0).connect 2023-11-17T05:21:25.813 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:19.764+0000 7fc17c5c7700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fc1740fb480 con 0x7fc1740faae0 2023-11-17T05:21:25.813 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:19.764+0000 7fc17a363700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fc1740faae0 0x7fc1740faf00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:21:25.813 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:19.764+0000 7fc17a363700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fc1740faae0 0x7fc1740faf00 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:35704/0 (socket says 172.21.15.40:35704) 2023-11-17T05:21:25.813 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:19.764+0000 7fc17a363700 1 -- 172.21.15.40:0/611473287 learned_addr learned my addr 172.21.15.40:0/611473287 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-11-17T05:21:25.813 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:19.765+0000 7fc17a363700 1 -- 172.21.15.40:0/611473287 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fc1740fb5c0 con 0x7fc1740faae0 2023-11-17T05:21:25.813 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:19.765+0000 7fc17a363700 1 --2- 172.21.15.40:0/611473287 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fc1740faae0 0x7fc1740faf00 secure :-1 s=READY pgs=1 cs=0 l=1 rev1=1 rx=0x7fc16400b5f0 tx=0x7fc164006ee0).ready entity=mon.0 client_cookie=9fdcb2c8254ab6fb server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:21:25.813 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:19.765+0000 7fc179361700 1 -- 172.21.15.40:0/611473287 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7fc16400d600 con 0x7fc1740faae0 2023-11-17T05:21:25.813 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:19.765+0000 7fc179361700 1 -- 172.21.15.40:0/611473287 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(25 keys) v1 ==== 1010+0+0 (secure 0 0 0) 0x7fc16400dbd0 con 0x7fc1740faae0 2023-11-17T05:21:25.813 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:19.765+0000 7fc179361700 1 -- 172.21.15.40:0/611473287 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7fc164011c00 con 0x7fc1740faae0 2023-11-17T05:21:25.813 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:19.765+0000 7fc17c5c7700 1 -- 172.21.15.40:0/611473287 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fc1740faae0 msgr2=0x7fc1740faf00 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:21:25.813 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:19.765+0000 7fc17c5c7700 1 --2- 172.21.15.40:0/611473287 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fc1740faae0 0x7fc1740faf00 secure :-1 s=READY pgs=1 cs=0 l=1 rev1=1 rx=0x7fc16400b5f0 tx=0x7fc164006ee0).stop 2023-11-17T05:21:25.813 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:19.766+0000 7fc17c5c7700 1 -- 172.21.15.40:0/611473287 shutdown_connections 2023-11-17T05:21:25.814 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:19.766+0000 7fc17c5c7700 1 --2- 172.21.15.40:0/611473287 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fc1740faae0 0x7fc1740faf00 unknown :-1 s=CLOSED pgs=1 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:21:25.814 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:19.766+0000 7fc17c5c7700 1 -- 172.21.15.40:0/611473287 >> 172.21.15.40:0/611473287 conn(0x7fc1740f6280 msgr2=0x7fc1740f86a0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:21:25.814 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:19.766+0000 7fc17c5c7700 1 -- 172.21.15.40:0/611473287 shutdown_connections 2023-11-17T05:21:25.814 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:19.766+0000 7fc17c5c7700 1 -- 172.21.15.40:0/611473287 wait complete. 2023-11-17T05:21:25.814 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:19.767+0000 7fc17c5c7700 1 Processor -- start 2023-11-17T05:21:25.814 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:19.767+0000 7fc17c5c7700 1 -- start start 2023-11-17T05:21:25.814 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:19.767+0000 7fc17c5c7700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fc1740faae0 0x7fc17410d720 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:21:25.814 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:19.767+0000 7fc17c5c7700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fc17406fcb0 con 0x7fc1740faae0 2023-11-17T05:21:25.814 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:19.767+0000 7fc17a363700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fc1740faae0 0x7fc17410d720 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:21:25.814 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:19.767+0000 7fc17a363700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fc1740faae0 0x7fc17410d720 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:35706/0 (socket says 172.21.15.40:35706) 2023-11-17T05:21:25.814 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:19.767+0000 7fc17a363700 1 -- 172.21.15.40:0/3180933970 learned_addr learned my addr 172.21.15.40:0/3180933970 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-11-17T05:21:25.814 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:19.768+0000 7fc17a363700 1 -- 172.21.15.40:0/3180933970 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fc16400b200 con 0x7fc1740faae0 2023-11-17T05:21:25.814 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:19.768+0000 7fc17a363700 1 --2- 172.21.15.40:0/3180933970 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fc1740faae0 0x7fc17410d720 secure :-1 s=READY pgs=2 cs=0 l=1 rev1=1 rx=0x7fc164002f40 tx=0x7fc164006ee0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:21:25.815 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:19.768+0000 7fc162ffd700 1 -- 172.21.15.40:0/3180933970 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7fc164011690 con 0x7fc1740faae0 2023-11-17T05:21:25.815 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:19.768+0000 7fc162ffd700 1 -- 172.21.15.40:0/3180933970 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(25 keys) v1 ==== 1010+0+0 (secure 0 0 0) 0x7fc16401b880 con 0x7fc1740faae0 2023-11-17T05:21:25.815 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:19.768+0000 7fc17c5c7700 1 -- 172.21.15.40:0/3180933970 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fc17406feb0 con 0x7fc1740faae0 2023-11-17T05:21:25.815 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:19.768+0000 7fc162ffd700 1 -- 172.21.15.40:0/3180933970 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7fc16401a6b0 con 0x7fc1740faae0 2023-11-17T05:21:25.815 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:19.768+0000 7fc17c5c7700 1 -- 172.21.15.40:0/3180933970 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fc17410ddd0 con 0x7fc1740faae0 2023-11-17T05:21:25.815 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:19.768+0000 7fc162ffd700 1 -- 172.21.15.40:0/3180933970 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 1) v1 ==== 484+0+0 (secure 0 0 0) 0x7fc16400f070 con 0x7fc1740faae0 2023-11-17T05:21:25.815 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:19.768+0000 7fc162ffd700 1 -- 172.21.15.40:0/3180933970 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(1..1 src has 1..1) v4 ==== 754+0+0 (secure 0 0 0) 0x7fc1640259f0 con 0x7fc1740faae0 2023-11-17T05:21:25.815 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:19.769+0000 7fc17c5c7700 1 -- 172.21.15.40:0/3180933970 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fc174045ad0 con 0x7fc1740faae0 2023-11-17T05:21:25.815 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:19.771+0000 7fc162ffd700 1 -- 172.21.15.40:0/3180933970 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+72097 (secure 0 0 0) 0x7fc164016070 con 0x7fc1740faae0 2023-11-17T05:21:25.815 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:19.840+0000 7fc17c5c7700 1 -- 172.21.15.40:0/3180933970 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command([{prefix=config set, name=public_network}] v 0) v1 -- 0x7fc174027a00 con 0x7fc1740faae0 2023-11-17T05:21:25.815 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:19.845+0000 7fc162ffd700 1 -- 172.21.15.40:0/3180933970 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mon_command_ack([{prefix=config set, name=public_network}]=0 v3)=0 v3) v1 ==== 124+0+0 (secure 0 0 0) 0x7fc16401bd60 con 0x7fc1740faae0 2023-11-17T05:21:25.815 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:19.846+0000 7fc17c5c7700 1 -- 172.21.15.40:0/3180933970 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fc1740faae0 msgr2=0x7fc17410d720 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:21:25.815 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:19.846+0000 7fc17c5c7700 1 --2- 172.21.15.40:0/3180933970 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fc1740faae0 0x7fc17410d720 secure :-1 s=READY pgs=2 cs=0 l=1 rev1=1 rx=0x7fc164002f40 tx=0x7fc164006ee0).stop 2023-11-17T05:21:25.816 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:19.846+0000 7fc17c5c7700 1 -- 172.21.15.40:0/3180933970 shutdown_connections 2023-11-17T05:21:25.816 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:19.846+0000 7fc17c5c7700 1 --2- 172.21.15.40:0/3180933970 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fc1740faae0 0x7fc17410d720 unknown :-1 s=CLOSED pgs=2 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:21:25.816 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:19.846+0000 7fc17c5c7700 1 -- 172.21.15.40:0/3180933970 >> 172.21.15.40:0/3180933970 conn(0x7fc1740f6280 msgr2=0x7fc1740f7e70 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:21:25.816 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:19.846+0000 7fc17c5c7700 1 -- 172.21.15.40:0/3180933970 shutdown_connections 2023-11-17T05:21:25.816 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:19.846+0000 7fc17c5c7700 1 -- 172.21.15.40:0/3180933970 wait complete. 2023-11-17T05:21:25.816 INFO:teuthology.orchestra.run.smithi040.stderr:Wrote config to /etc/ceph/ceph.conf 2023-11-17T05:21:25.816 INFO:teuthology.orchestra.run.smithi040.stderr:Wrote keyring to /etc/ceph/ceph.client.admin.keyring 2023-11-17T05:21:25.816 INFO:teuthology.orchestra.run.smithi040.stderr:Creating mgr... 2023-11-17T05:21:25.816 INFO:teuthology.orchestra.run.smithi040.stderr:Verifying port 9283 ... 2023-11-17T05:21:25.816 INFO:teuthology.orchestra.run.smithi040.stderr:Running command: systemctl daemon-reload 2023-11-17T05:21:25.816 INFO:teuthology.orchestra.run.smithi040.stderr:Running command: systemctl stop ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7@mgr.smithi040.ogcprl 2023-11-17T05:21:25.816 INFO:teuthology.orchestra.run.smithi040.stderr:Running command: systemctl reset-failed ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7@mgr.smithi040.ogcprl 2023-11-17T05:21:25.816 INFO:teuthology.orchestra.run.smithi040.stderr:systemctl: Failed to reset failed state of unit ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7@mgr.smithi040.ogcprl.service: Unit ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7@mgr.smithi040.ogcprl.service not loaded. 2023-11-17T05:21:25.816 INFO:teuthology.orchestra.run.smithi040.stderr:Running command: systemctl enable ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7@mgr.smithi040.ogcprl 2023-11-17T05:21:25.817 INFO:teuthology.orchestra.run.smithi040.stderr:systemctl: Created symlink /etc/systemd/system/ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7.target.wants/ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7@mgr.smithi040.ogcprl.service → /etc/systemd/system/ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7@.service. 2023-11-17T05:21:25.817 INFO:teuthology.orchestra.run.smithi040.stderr:Running command: systemctl start ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7@mgr.smithi040.ogcprl 2023-11-17T05:21:25.817 INFO:teuthology.orchestra.run.smithi040.stderr:Running command: systemctl is-enabled firewalld.service 2023-11-17T05:21:25.817 INFO:teuthology.orchestra.run.smithi040.stderr:systemctl: disabled 2023-11-17T05:21:25.817 INFO:teuthology.orchestra.run.smithi040.stderr:Running command: systemctl is-active firewalld.service 2023-11-17T05:21:25.817 INFO:teuthology.orchestra.run.smithi040.stderr:systemctl: inactive 2023-11-17T05:21:25.817 INFO:teuthology.orchestra.run.smithi040.stderr:firewalld.service is not enabled 2023-11-17T05:21:25.817 INFO:teuthology.orchestra.run.smithi040.stderr:Not possible to enable service . firewalld.service is not available 2023-11-17T05:21:25.817 INFO:teuthology.orchestra.run.smithi040.stderr:Not possible to open ports <[]>. firewalld.service is not available 2023-11-17T05:21:25.817 INFO:teuthology.orchestra.run.smithi040.stderr:Running command: systemctl is-enabled firewalld.service 2023-11-17T05:21:25.817 INFO:teuthology.orchestra.run.smithi040.stderr:systemctl: disabled 2023-11-17T05:21:25.817 INFO:teuthology.orchestra.run.smithi040.stderr:Running command: systemctl is-active firewalld.service 2023-11-17T05:21:25.817 INFO:teuthology.orchestra.run.smithi040.stderr:systemctl: inactive 2023-11-17T05:21:25.817 INFO:teuthology.orchestra.run.smithi040.stderr:firewalld.service is not enabled 2023-11-17T05:21:25.818 INFO:teuthology.orchestra.run.smithi040.stderr:Not possible to open ports <[9283]>. firewalld.service is not available 2023-11-17T05:21:25.818 INFO:teuthology.orchestra.run.smithi040.stderr:Waiting for mgr to start... 2023-11-17T05:21:25.818 INFO:teuthology.orchestra.run.smithi040.stderr:Waiting for mgr... 2023-11-17T05:21:25.818 INFO:teuthology.orchestra.run.smithi040.stderr:Running command: /bin/podman run --rm --ipc=host --no-hosts --net=host --entrypoint /usr/bin/ceph --init -e CONTAINER_IMAGE=docker.io/ceph/ceph:v16.2.4 -e NODE_NAME=smithi040 -e CEPH_USE_RANDOM_NONCE=1 -v /var/log/ceph/ed6470e2-8508-11ee-8dbe-212e2dc638e7:/var/log/ceph:z -v /tmp/ceph-tmpzx79_7_h:/etc/ceph/ceph.client.admin.keyring:z -v /tmp/ceph-tmpru9tjirw:/etc/ceph/ceph.conf:z docker.io/ceph/ceph:v16.2.4 status -f json-pretty 2023-11-17T05:21:25.818 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:20 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:20:19.847297+0000 mon.smithi040 (mon.0) 6 : audit [INF] from='client.? 172.21.15.40:0/3180933970' entity='client.admin' 2023-11-17T05:21:25.827 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:21.243+0000 7f602a779700 1 Processor -- start 2023-11-17T05:21:25.827 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:21.243+0000 7f602a779700 1 -- start start 2023-11-17T05:21:25.827 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:21.244+0000 7f602a779700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f60240faae0 0x7f60240faf00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 rx=0 tx=0).connect 2023-11-17T05:21:25.827 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:21.244+0000 7f602a779700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f60240fb480 con 0x7f60240faae0 2023-11-17T05:21:25.827 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:21.244+0000 7f6023fff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f60240faae0 0x7f60240faf00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:21:25.827 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:21.244+0000 7f6023fff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f60240faae0 0x7f60240faf00 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:35734/0 (socket says 172.21.15.40:35734) 2023-11-17T05:21:25.827 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:21.244+0000 7f6023fff700 1 -- 172.21.15.40:0/1939962767 learned_addr learned my addr 172.21.15.40:0/1939962767 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-11-17T05:21:25.827 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:21.245+0000 7f6023fff700 1 -- 172.21.15.40:0/1939962767 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f60240fb5c0 con 0x7f60240faae0 2023-11-17T05:21:25.827 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:21.245+0000 7f6023fff700 1 --2- 172.21.15.40:0/1939962767 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f60240faae0 0x7f60240faf00 secure :-1 s=READY pgs=5 cs=0 l=1 rev1=1 rx=0x7f601400b5f0 tx=0x7f6014006ee0).ready entity=mon.0 client_cookie=e6b6d1fa3e4fa0c server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:21:25.828 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:21.245+0000 7f6022ffd700 1 -- 172.21.15.40:0/1939962767 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f601400d600 con 0x7f60240faae0 2023-11-17T05:21:25.828 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:21.245+0000 7f6022ffd700 1 -- 172.21.15.40:0/1939962767 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(25 keys) v1 ==== 1010+0+0 (secure 0 0 0) 0x7f601400dbd0 con 0x7f60240faae0 2023-11-17T05:21:25.828 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:21.245+0000 7f6022ffd700 1 -- 172.21.15.40:0/1939962767 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f6014011c00 con 0x7f60240faae0 2023-11-17T05:21:25.828 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:21.246+0000 7f602a779700 1 -- 172.21.15.40:0/1939962767 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f60240faae0 msgr2=0x7f60240faf00 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:21:25.828 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:21.246+0000 7f602a779700 1 --2- 172.21.15.40:0/1939962767 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f60240faae0 0x7f60240faf00 secure :-1 s=READY pgs=5 cs=0 l=1 rev1=1 rx=0x7f601400b5f0 tx=0x7f6014006ee0).stop 2023-11-17T05:21:25.828 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:21.246+0000 7f602a779700 1 -- 172.21.15.40:0/1939962767 shutdown_connections 2023-11-17T05:21:25.828 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:21.246+0000 7f602a779700 1 --2- 172.21.15.40:0/1939962767 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f60240faae0 0x7f60240faf00 unknown :-1 s=CLOSED pgs=5 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:21:25.828 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:21.246+0000 7f602a779700 1 -- 172.21.15.40:0/1939962767 >> 172.21.15.40:0/1939962767 conn(0x7f60240f6280 msgr2=0x7f60240f86a0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:21:25.828 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:21.246+0000 7f602a779700 1 -- 172.21.15.40:0/1939962767 shutdown_connections 2023-11-17T05:21:25.828 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:21.246+0000 7f602a779700 1 -- 172.21.15.40:0/1939962767 wait complete. 2023-11-17T05:21:25.828 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:21.246+0000 7f602a779700 1 Processor -- start 2023-11-17T05:21:25.828 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:21.247+0000 7f602a779700 1 -- start start 2023-11-17T05:21:25.828 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:21.247+0000 7f602a779700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f60240faae0 0x7f602410d730 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:21:25.829 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:21.247+0000 7f602a779700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f602406fc00 con 0x7f60240faae0 2023-11-17T05:21:25.829 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:21.247+0000 7f6023fff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f60240faae0 0x7f602410d730 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:21:25.829 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:21.247+0000 7f6023fff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f60240faae0 0x7f602410d730 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:35738/0 (socket says 172.21.15.40:35738) 2023-11-17T05:21:25.829 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:21.247+0000 7f6023fff700 1 -- 172.21.15.40:0/202276924 learned_addr learned my addr 172.21.15.40:0/202276924 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-11-17T05:21:25.829 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:21.248+0000 7f6023fff700 1 -- 172.21.15.40:0/202276924 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f601400b200 con 0x7f60240faae0 2023-11-17T05:21:25.829 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:21.248+0000 7f6023fff700 1 --2- 172.21.15.40:0/202276924 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f60240faae0 0x7f602410d730 secure :-1 s=READY pgs=6 cs=0 l=1 rev1=1 rx=0x7f6014002f40 tx=0x7f6014006ee0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:21:25.829 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:21.248+0000 7f6020ff9700 1 -- 172.21.15.40:0/202276924 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f6014011690 con 0x7f60240faae0 2023-11-17T05:21:25.829 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:21.248+0000 7f6020ff9700 1 -- 172.21.15.40:0/202276924 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(25 keys) v1 ==== 1010+0+0 (secure 0 0 0) 0x7f601401b880 con 0x7f60240faae0 2023-11-17T05:21:25.829 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:21.248+0000 7f602a779700 1 -- 172.21.15.40:0/202276924 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f602406fe00 con 0x7f60240faae0 2023-11-17T05:21:25.829 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:21.248+0000 7f6020ff9700 1 -- 172.21.15.40:0/202276924 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f601401a6b0 con 0x7f60240faae0 2023-11-17T05:21:25.829 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:21.248+0000 7f602a779700 1 -- 172.21.15.40:0/202276924 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f602410dde0 con 0x7f60240faae0 2023-11-17T05:21:25.829 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:21.248+0000 7f6020ff9700 1 -- 172.21.15.40:0/202276924 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 1) v1 ==== 484+0+0 (secure 0 0 0) 0x7f601400f070 con 0x7f60240faae0 2023-11-17T05:21:25.829 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:21.248+0000 7f6020ff9700 1 -- 172.21.15.40:0/202276924 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(1..1 src has 1..1) v4 ==== 754+0+0 (secure 0 0 0) 0x7f6014025a00 con 0x7f60240faae0 2023-11-17T05:21:25.829 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:21.250+0000 7f602a779700 1 -- 172.21.15.40:0/202276924 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f6024045ad0 con 0x7f60240faae0 2023-11-17T05:21:25.830 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:21.252+0000 7f6020ff9700 1 -- 172.21.15.40:0/202276924 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+72097 (secure 0 0 0) 0x7f6014016070 con 0x7f60240faae0 2023-11-17T05:21:25.830 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:21.326+0000 7f602a779700 1 -- 172.21.15.40:0/202276924 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "status", "format": "json-pretty"} v 0) v1 -- 0x7f6024027a00 con 0x7f60240faae0 2023-11-17T05:21:25.830 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:21.327+0000 7f6020ff9700 1 -- 172.21.15.40:0/202276924 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mon_command_ack([{"prefix": "status", "format": "json-pretty"}]=0 v0) v1 ==== 79+0+1230 (secure 0 0 0) 0x7f601401ad20 con 0x7f60240faae0 2023-11-17T05:21:25.830 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:25.830 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: { 2023-11-17T05:21:25.830 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: "fsid": "ed6470e2-8508-11ee-8dbe-212e2dc638e7", 2023-11-17T05:21:25.830 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: "health": { 2023-11-17T05:21:25.830 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: "status": "HEALTH_OK", 2023-11-17T05:21:25.830 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: "checks": {}, 2023-11-17T05:21:25.830 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: "mutes": [] 2023-11-17T05:21:25.830 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: }, 2023-11-17T05:21:25.830 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: "election_epoch": 5, 2023-11-17T05:21:25.830 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: "quorum": [ 2023-11-17T05:21:25.831 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 0 2023-11-17T05:21:25.831 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: ], 2023-11-17T05:21:25.831 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: "quorum_names": [ 2023-11-17T05:21:25.831 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: "smithi040" 2023-11-17T05:21:25.831 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: ], 2023-11-17T05:21:25.831 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: "quorum_age": 1, 2023-11-17T05:21:25.831 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: "monmap": { 2023-11-17T05:21:25.831 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: "epoch": 1, 2023-11-17T05:21:25.831 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: "min_mon_release_name": "pacific", 2023-11-17T05:21:25.831 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: "num_mons": 1 2023-11-17T05:21:25.831 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: }, 2023-11-17T05:21:25.831 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: "osdmap": { 2023-11-17T05:21:25.831 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: "epoch": 1, 2023-11-17T05:21:25.831 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: "num_osds": 0, 2023-11-17T05:21:25.832 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: "num_up_osds": 0, 2023-11-17T05:21:25.832 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: "osd_up_since": 0, 2023-11-17T05:21:25.832 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: "num_in_osds": 0, 2023-11-17T05:21:25.832 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: "osd_in_since": 0, 2023-11-17T05:21:25.832 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: "num_remapped_pgs": 0 2023-11-17T05:21:25.832 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: }, 2023-11-17T05:21:25.832 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: "pgmap": { 2023-11-17T05:21:25.832 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: "pgs_by_state": [], 2023-11-17T05:21:25.832 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: "num_pgs": 0, 2023-11-17T05:21:25.832 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: "num_pools": 0, 2023-11-17T05:21:25.832 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: "num_objects": 0, 2023-11-17T05:21:25.832 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: "data_bytes": 0, 2023-11-17T05:21:25.832 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: "bytes_used": 0, 2023-11-17T05:21:25.832 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: "bytes_avail": 0, 2023-11-17T05:21:25.832 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: "bytes_total": 0 2023-11-17T05:21:25.833 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: }, 2023-11-17T05:21:25.833 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: "fsmap": { 2023-11-17T05:21:25.833 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: "epoch": 1, 2023-11-17T05:21:25.833 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: "by_rank": [], 2023-11-17T05:21:25.833 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: "up:standby": 0 2023-11-17T05:21:25.833 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: }, 2023-11-17T05:21:25.833 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: "mgrmap": { 2023-11-17T05:21:25.833 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: "available": false, 2023-11-17T05:21:25.833 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: "num_standbys": 0, 2023-11-17T05:21:25.833 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: "modules": [ 2023-11-17T05:21:25.833 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: "iostat", 2023-11-17T05:21:25.833 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: "restful" 2023-11-17T05:21:25.833 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: ], 2023-11-17T05:21:25.833 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: "services": {} 2023-11-17T05:21:25.834 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: }, 2023-11-17T05:21:25.834 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: "servicemap": { 2023-11-17T05:21:25.834 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: "epoch": 1, 2023-11-17T05:21:25.834 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: "modified": "2023-11-17T05:20:15.855428+0000", 2023-11-17T05:21:25.834 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: "services": {} 2023-11-17T05:21:25.834 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: }, 2023-11-17T05:21:25.834 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: "progress_events": {} 2023-11-17T05:21:25.834 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: } 2023-11-17T05:21:25.834 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:21.328+0000 7f602a779700 1 -- 172.21.15.40:0/202276924 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f60240faae0 msgr2=0x7f602410d730 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:21:25.834 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:21.328+0000 7f602a779700 1 --2- 172.21.15.40:0/202276924 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f60240faae0 0x7f602410d730 secure :-1 s=READY pgs=6 cs=0 l=1 rev1=1 rx=0x7f6014002f40 tx=0x7f6014006ee0).stop 2023-11-17T05:21:25.834 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:21.328+0000 7f602a779700 1 -- 172.21.15.40:0/202276924 shutdown_connections 2023-11-17T05:21:25.834 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:21.328+0000 7f602a779700 1 --2- 172.21.15.40:0/202276924 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f60240faae0 0x7f602410d730 unknown :-1 s=CLOSED pgs=6 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:21:25.834 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:21.328+0000 7f602a779700 1 -- 172.21.15.40:0/202276924 >> 172.21.15.40:0/202276924 conn(0x7f60240f6280 msgr2=0x7f60240f7ef0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:21:25.834 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:21.328+0000 7f602a779700 1 -- 172.21.15.40:0/202276924 shutdown_connections 2023-11-17T05:21:25.835 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:21.328+0000 7f602a779700 1 -- 172.21.15.40:0/202276924 wait complete. 2023-11-17T05:21:25.835 INFO:teuthology.orchestra.run.smithi040.stderr:mgr not available, waiting (1/15)... 2023-11-17T05:21:25.835 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:21 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:20:21.328582+0000 mon.smithi040 (mon.0) 7 : audit [DBG] from='client.? 172.21.15.40:0/202276924' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2023-11-17T05:21:25.849 INFO:teuthology.orchestra.run.smithi040.stderr:Running command: /bin/podman run --rm --ipc=host --no-hosts --net=host --entrypoint /usr/bin/ceph --init -e CONTAINER_IMAGE=docker.io/ceph/ceph:v16.2.4 -e NODE_NAME=smithi040 -e CEPH_USE_RANDOM_NONCE=1 -v /var/log/ceph/ed6470e2-8508-11ee-8dbe-212e2dc638e7:/var/log/ceph:z -v /tmp/ceph-tmpzx79_7_h:/etc/ceph/ceph.client.admin.keyring:z -v /tmp/ceph-tmpru9tjirw:/etc/ceph/ceph.conf:z docker.io/ceph/ceph:v16.2.4 status -f json-pretty 2023-11-17T05:21:25.849 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:24.331+0000 7f56c8f4c700 1 Processor -- start 2023-11-17T05:21:25.849 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:24.331+0000 7f56c8f4c700 1 -- start start 2023-11-17T05:21:25.849 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:24.332+0000 7f56c8f4c700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f56c406fcd0 0x7f56c40fe7a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 rx=0 tx=0).connect 2023-11-17T05:21:25.850 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:24.332+0000 7f56c8f4c700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f56c40fed20 con 0x7f56c406fcd0 2023-11-17T05:21:25.850 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:24.332+0000 7f56c259c700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f56c406fcd0 0x7f56c40fe7a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:21:25.850 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:24.332+0000 7f56c259c700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f56c406fcd0 0x7f56c40fe7a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:35754/0 (socket says 172.21.15.40:35754) 2023-11-17T05:21:25.850 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:24.332+0000 7f56c259c700 1 -- 172.21.15.40:0/1587129538 learned_addr learned my addr 172.21.15.40:0/1587129538 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-11-17T05:21:25.850 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:24.332+0000 7f56c259c700 1 -- 172.21.15.40:0/1587129538 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f56c40fee60 con 0x7f56c406fcd0 2023-11-17T05:21:25.850 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:24.333+0000 7f56c259c700 1 --2- 172.21.15.40:0/1587129538 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f56c406fcd0 0x7f56c40fe7a0 secure :-1 s=READY pgs=7 cs=0 l=1 rev1=1 rx=0x7f56b400b5f0 tx=0x7f56b4006ee0).ready entity=mon.0 client_cookie=49afd72ac7f043f7 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:21:25.850 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:24.333+0000 7f56c159a700 1 -- 172.21.15.40:0/1587129538 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f56b400d600 con 0x7f56c406fcd0 2023-11-17T05:21:25.850 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:24.333+0000 7f56c159a700 1 -- 172.21.15.40:0/1587129538 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(25 keys) v1 ==== 1010+0+0 (secure 0 0 0) 0x7f56b400dbd0 con 0x7f56c406fcd0 2023-11-17T05:21:25.850 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:24.333+0000 7f56c159a700 1 -- 172.21.15.40:0/1587129538 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f56b4011c00 con 0x7f56c406fcd0 2023-11-17T05:21:25.850 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:24.334+0000 7f56c8f4c700 1 -- 172.21.15.40:0/1587129538 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f56c406fcd0 msgr2=0x7f56c40fe7a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:21:25.850 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:24.334+0000 7f56c8f4c700 1 --2- 172.21.15.40:0/1587129538 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f56c406fcd0 0x7f56c40fe7a0 secure :-1 s=READY pgs=7 cs=0 l=1 rev1=1 rx=0x7f56b400b5f0 tx=0x7f56b4006ee0).stop 2023-11-17T05:21:25.850 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:24.334+0000 7f56c8f4c700 1 -- 172.21.15.40:0/1587129538 shutdown_connections 2023-11-17T05:21:25.850 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:24.334+0000 7f56c8f4c700 1 --2- 172.21.15.40:0/1587129538 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f56c406fcd0 0x7f56c40fe7a0 unknown :-1 s=CLOSED pgs=7 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:21:25.851 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:24.334+0000 7f56c8f4c700 1 -- 172.21.15.40:0/1587129538 >> 172.21.15.40:0/1587129538 conn(0x7f56c40f6280 msgr2=0x7f56c40f86a0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:21:25.851 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:24.334+0000 7f56c8f4c700 1 -- 172.21.15.40:0/1587129538 shutdown_connections 2023-11-17T05:21:25.851 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:24.334+0000 7f56c8f4c700 1 -- 172.21.15.40:0/1587129538 wait complete. 2023-11-17T05:21:25.851 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:24.335+0000 7f56c8f4c700 1 Processor -- start 2023-11-17T05:21:25.851 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:24.335+0000 7f56c8f4c700 1 -- start start 2023-11-17T05:21:25.851 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:24.335+0000 7f56c8f4c700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f56c406fcd0 0x7f56c40fc220 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:21:25.851 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:24.335+0000 7f56c8f4c700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f56c40fa6a0 con 0x7f56c406fcd0 2023-11-17T05:21:25.851 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:24.335+0000 7f56c259c700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f56c406fcd0 0x7f56c40fc220 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:21:25.851 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:24.335+0000 7f56c259c700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f56c406fcd0 0x7f56c40fc220 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:35760/0 (socket says 172.21.15.40:35760) 2023-11-17T05:21:25.851 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:24.335+0000 7f56c259c700 1 -- 172.21.15.40:0/2893239704 learned_addr learned my addr 172.21.15.40:0/2893239704 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-11-17T05:21:25.851 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:24.336+0000 7f56c259c700 1 -- 172.21.15.40:0/2893239704 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f56b400b200 con 0x7f56c406fcd0 2023-11-17T05:21:25.851 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:24.336+0000 7f56c259c700 1 --2- 172.21.15.40:0/2893239704 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f56c406fcd0 0x7f56c40fc220 secure :-1 s=READY pgs=8 cs=0 l=1 rev1=1 rx=0x7f56b4000c00 tx=0x7f56b4006ee0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:21:25.851 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:24.336+0000 7f56b2ffd700 1 -- 172.21.15.40:0/2893239704 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f56b4011690 con 0x7f56c406fcd0 2023-11-17T05:21:25.851 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:24.336+0000 7f56b2ffd700 1 -- 172.21.15.40:0/2893239704 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(25 keys) v1 ==== 1010+0+0 (secure 0 0 0) 0x7f56b400d9f0 con 0x7f56c406fcd0 2023-11-17T05:21:25.852 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:24.336+0000 7f56c8f4c700 1 -- 172.21.15.40:0/2893239704 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f56c40fa840 con 0x7f56c406fcd0 2023-11-17T05:21:25.852 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:24.336+0000 7f56b2ffd700 1 -- 172.21.15.40:0/2893239704 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f56b401b710 con 0x7f56c406fcd0 2023-11-17T05:21:25.852 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:24.336+0000 7f56c8f4c700 1 -- 172.21.15.40:0/2893239704 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f56c40fac50 con 0x7f56c406fcd0 2023-11-17T05:21:25.852 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:24.336+0000 7f56b2ffd700 1 -- 172.21.15.40:0/2893239704 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 1) v1 ==== 484+0+0 (secure 0 0 0) 0x7f56b400f070 con 0x7f56c406fcd0 2023-11-17T05:21:25.852 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:24.336+0000 7f56b2ffd700 1 -- 172.21.15.40:0/2893239704 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(1..1 src has 1..1) v4 ==== 754+0+0 (secure 0 0 0) 0x7f56b4025a30 con 0x7f56c406fcd0 2023-11-17T05:21:25.852 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:24.337+0000 7f56c8f4c700 1 -- 172.21.15.40:0/2893239704 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f56c4045ad0 con 0x7f56c406fcd0 2023-11-17T05:21:25.852 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:24.340+0000 7f56b2ffd700 1 -- 172.21.15.40:0/2893239704 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+72097 (secure 0 0 0) 0x7f56b4016070 con 0x7f56c406fcd0 2023-11-17T05:21:25.852 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:24.415+0000 7f56c8f4c700 1 -- 172.21.15.40:0/2893239704 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "status", "format": "json-pretty"} v 0) v1 -- 0x7f56c4104820 con 0x7f56c406fcd0 2023-11-17T05:21:25.852 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:24.415+0000 7f56b2ffd700 1 -- 172.21.15.40:0/2893239704 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mon_command_ack([{"prefix": "status", "format": "json-pretty"}]=0 v0) v1 ==== 79+0+1230 (secure 0 0 0) 0x7f56b401aaf0 con 0x7f56c406fcd0 2023-11-17T05:21:25.852 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:25.852 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: { 2023-11-17T05:21:25.852 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: "fsid": "ed6470e2-8508-11ee-8dbe-212e2dc638e7", 2023-11-17T05:21:25.852 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: "health": { 2023-11-17T05:21:25.852 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: "status": "HEALTH_OK", 2023-11-17T05:21:25.853 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: "checks": {}, 2023-11-17T05:21:25.853 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: "mutes": [] 2023-11-17T05:21:25.853 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: }, 2023-11-17T05:21:25.853 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: "election_epoch": 5, 2023-11-17T05:21:25.853 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: "quorum": [ 2023-11-17T05:21:25.853 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 0 2023-11-17T05:21:25.853 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: ], 2023-11-17T05:21:25.853 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: "quorum_names": [ 2023-11-17T05:21:25.853 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: "smithi040" 2023-11-17T05:21:25.853 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: ], 2023-11-17T05:21:25.853 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: "quorum_age": 5, 2023-11-17T05:21:25.853 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: "monmap": { 2023-11-17T05:21:25.853 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: "epoch": 1, 2023-11-17T05:21:25.853 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: "min_mon_release_name": "pacific", 2023-11-17T05:21:25.854 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: "num_mons": 1 2023-11-17T05:21:25.854 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: }, 2023-11-17T05:21:25.854 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: "osdmap": { 2023-11-17T05:21:25.854 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: "epoch": 1, 2023-11-17T05:21:25.854 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: "num_osds": 0, 2023-11-17T05:21:25.854 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: "num_up_osds": 0, 2023-11-17T05:21:25.854 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: "osd_up_since": 0, 2023-11-17T05:21:25.854 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: "num_in_osds": 0, 2023-11-17T05:21:25.854 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: "osd_in_since": 0, 2023-11-17T05:21:25.854 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: "num_remapped_pgs": 0 2023-11-17T05:21:25.854 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: }, 2023-11-17T05:21:25.854 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: "pgmap": { 2023-11-17T05:21:25.854 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: "pgs_by_state": [], 2023-11-17T05:21:25.854 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: "num_pgs": 0, 2023-11-17T05:21:25.855 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: "num_pools": 0, 2023-11-17T05:21:25.855 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: "num_objects": 0, 2023-11-17T05:21:25.855 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: "data_bytes": 0, 2023-11-17T05:21:25.855 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: "bytes_used": 0, 2023-11-17T05:21:25.855 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: "bytes_avail": 0, 2023-11-17T05:21:25.855 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: "bytes_total": 0 2023-11-17T05:21:25.855 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: }, 2023-11-17T05:21:25.855 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: "fsmap": { 2023-11-17T05:21:25.855 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: "epoch": 1, 2023-11-17T05:21:25.855 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: "by_rank": [], 2023-11-17T05:21:25.855 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: "up:standby": 0 2023-11-17T05:21:25.855 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: }, 2023-11-17T05:21:25.855 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: "mgrmap": { 2023-11-17T05:21:25.855 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: "available": false, 2023-11-17T05:21:25.856 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: "num_standbys": 0, 2023-11-17T05:21:25.856 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: "modules": [ 2023-11-17T05:21:25.856 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: "iostat", 2023-11-17T05:21:25.856 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: "restful" 2023-11-17T05:21:25.856 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: ], 2023-11-17T05:21:25.856 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: "services": {} 2023-11-17T05:21:25.856 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: }, 2023-11-17T05:21:25.856 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: "servicemap": { 2023-11-17T05:21:25.856 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: "epoch": 1, 2023-11-17T05:21:25.856 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: "modified": "2023-11-17T05:20:15.855428+0000", 2023-11-17T05:21:25.856 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: "services": {} 2023-11-17T05:21:25.856 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: }, 2023-11-17T05:21:25.856 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: "progress_events": {} 2023-11-17T05:21:25.856 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: } 2023-11-17T05:21:25.857 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:24.416+0000 7f56c8f4c700 1 -- 172.21.15.40:0/2893239704 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f56c406fcd0 msgr2=0x7f56c40fc220 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:21:25.857 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:24.416+0000 7f56c8f4c700 1 --2- 172.21.15.40:0/2893239704 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f56c406fcd0 0x7f56c40fc220 secure :-1 s=READY pgs=8 cs=0 l=1 rev1=1 rx=0x7f56b4000c00 tx=0x7f56b4006ee0).stop 2023-11-17T05:21:25.857 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:24.416+0000 7f56c8f4c700 1 -- 172.21.15.40:0/2893239704 shutdown_connections 2023-11-17T05:21:25.857 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:24.416+0000 7f56c8f4c700 1 --2- 172.21.15.40:0/2893239704 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f56c406fcd0 0x7f56c40fc220 unknown :-1 s=CLOSED pgs=8 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:21:25.857 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:24.416+0000 7f56c8f4c700 1 -- 172.21.15.40:0/2893239704 >> 172.21.15.40:0/2893239704 conn(0x7f56c40f6280 msgr2=0x7f56c40f7e70 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:21:25.857 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:24.416+0000 7f56c8f4c700 1 -- 172.21.15.40:0/2893239704 shutdown_connections 2023-11-17T05:21:25.857 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:24.416+0000 7f56c8f4c700 1 -- 172.21.15.40:0/2893239704 wait complete. 2023-11-17T05:21:25.857 INFO:teuthology.orchestra.run.smithi040.stderr:mgr not available, waiting (2/15)... 2023-11-17T05:21:25.857 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:25 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:20:24.417182+0000 mon.smithi040 (mon.0) 8 : audit [DBG] from='client.? 172.21.15.40:0/2893239704' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2023-11-17T05:21:25.858 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:26 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:20:26.410676+0000 mon.smithi040 (mon.0) 9 : cluster [INF] Activating manager daemon smithi040.ogcprl 2023-11-17T05:21:25.858 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:26 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:20:26.413439+0000 mon.smithi040 (mon.0) 10 : cluster [DBG] mgrmap e2: smithi040.ogcprl(active, starting, since 0.00290327s) 2023-11-17T05:21:25.858 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:26 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:20:26.414604+0000 mon.smithi040 (mon.0) 11 : audit [DBG] from='mgr.14100 172.21.15.40:0/1928545105' entity='mgr.smithi040.ogcprl' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-11-17T05:21:25.858 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:26 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:20:26.414824+0000 mon.smithi040 (mon.0) 12 : audit [DBG] from='mgr.14100 172.21.15.40:0/1928545105' entity='mgr.smithi040.ogcprl' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-11-17T05:21:25.858 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:26 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:20:26.415009+0000 mon.smithi040 (mon.0) 13 : audit [DBG] from='mgr.14100 172.21.15.40:0/1928545105' entity='mgr.smithi040.ogcprl' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-11-17T05:21:25.858 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:26 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:20:26.415215+0000 mon.smithi040 (mon.0) 14 : audit [DBG] from='mgr.14100 172.21.15.40:0/1928545105' entity='mgr.smithi040.ogcprl' cmd=[{"prefix": "mon metadata", "id": "smithi040"}]: dispatch 2023-11-17T05:21:25.858 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:26 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:20:26.415523+0000 mon.smithi040 (mon.0) 15 : audit [DBG] from='mgr.14100 172.21.15.40:0/1928545105' entity='mgr.smithi040.ogcprl' cmd=[{"prefix": "mgr metadata", "who": "smithi040.ogcprl", "id": "smithi040.ogcprl"}]: dispatch 2023-11-17T05:21:25.858 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:26 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:20:26.429176+0000 mon.smithi040 (mon.0) 16 : cluster [INF] Manager daemon smithi040.ogcprl is now available 2023-11-17T05:21:25.858 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:26 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:20:26.438302+0000 mon.smithi040 (mon.0) 17 : audit [INF] from='mgr.14100 172.21.15.40:0/1928545105' entity='mgr.smithi040.ogcprl' 2023-11-17T05:21:25.858 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:26 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:20:26.443252+0000 mon.smithi040 (mon.0) 18 : audit [INF] from='mgr.14100 172.21.15.40:0/1928545105' entity='mgr.smithi040.ogcprl' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi040.ogcprl/mirror_snapshot_schedule"}]: dispatch 2023-11-17T05:21:25.858 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:26 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:20:26.444559+0000 mon.smithi040 (mon.0) 19 : audit [INF] from='mgr.14100 172.21.15.40:0/1928545105' entity='mgr.smithi040.ogcprl' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi040.ogcprl/trash_purge_schedule"}]: dispatch 2023-11-17T05:21:25.858 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:26 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:20:26.449450+0000 mon.smithi040 (mon.0) 20 : audit [INF] from='mgr.14100 172.21.15.40:0/1928545105' entity='mgr.smithi040.ogcprl' 2023-11-17T05:21:25.859 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:26 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:20:26.451880+0000 mon.smithi040 (mon.0) 21 : audit [INF] from='mgr.14100 172.21.15.40:0/1928545105' entity='mgr.smithi040.ogcprl' 2023-11-17T05:21:25.868 INFO:teuthology.orchestra.run.smithi040.stderr:Running command: /bin/podman run --rm --ipc=host --no-hosts --net=host --entrypoint /usr/bin/ceph --init -e CONTAINER_IMAGE=docker.io/ceph/ceph:v16.2.4 -e NODE_NAME=smithi040 -e CEPH_USE_RANDOM_NONCE=1 -v /var/log/ceph/ed6470e2-8508-11ee-8dbe-212e2dc638e7:/var/log/ceph:z -v /tmp/ceph-tmpzx79_7_h:/etc/ceph/ceph.client.admin.keyring:z -v /tmp/ceph-tmpru9tjirw:/etc/ceph/ceph.conf:z docker.io/ceph/ceph:v16.2.4 status -f json-pretty 2023-11-17T05:21:25.868 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:27.142+0000 7f61dcae3700 1 Processor -- start 2023-11-17T05:21:25.868 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:27.142+0000 7f61dcae3700 1 -- start start 2023-11-17T05:21:25.868 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:27.142+0000 7f61dcae3700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f61d80fa8f0 0x7f61d80fad10 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 rx=0 tx=0).connect 2023-11-17T05:21:25.868 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:27.142+0000 7f61dcae3700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f61d80fb200 con 0x7f61d80fa8f0 2023-11-17T05:21:25.868 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:27.143+0000 7f61d77fe700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f61d80fa8f0 0x7f61d80fad10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:21:25.868 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:27.143+0000 7f61d77fe700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f61d80fa8f0 0x7f61d80fad10 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:54306/0 (socket says 172.21.15.40:54306) 2023-11-17T05:21:25.868 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:27.143+0000 7f61d77fe700 1 -- 172.21.15.40:0/21794375 learned_addr learned my addr 172.21.15.40:0/21794375 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-11-17T05:21:25.868 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:27.143+0000 7f61d77fe700 1 -- 172.21.15.40:0/21794375 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f61d80fb340 con 0x7f61d80fa8f0 2023-11-17T05:21:25.868 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:27.143+0000 7f61d77fe700 1 --2- 172.21.15.40:0/21794375 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f61d80fa8f0 0x7f61d80fad10 secure :-1 s=READY pgs=14 cs=0 l=1 rev1=1 rx=0x7f61c800b5f0 tx=0x7f61c8006ee0).ready entity=mon.0 client_cookie=6acb8b26870ca26b server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:21:25.868 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:27.143+0000 7f61d67fc700 1 -- 172.21.15.40:0/21794375 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f61c800d600 con 0x7f61d80fa8f0 2023-11-17T05:21:25.869 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:27.143+0000 7f61d67fc700 1 -- 172.21.15.40:0/21794375 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(25 keys) v1 ==== 1010+0+0 (secure 0 0 0) 0x7f61c800dbd0 con 0x7f61d80fa8f0 2023-11-17T05:21:25.869 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:27.143+0000 7f61d67fc700 1 -- 172.21.15.40:0/21794375 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f61c8011c00 con 0x7f61d80fa8f0 2023-11-17T05:21:25.869 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:27.144+0000 7f61dcae3700 1 -- 172.21.15.40:0/21794375 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f61d80fa8f0 msgr2=0x7f61d80fad10 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:21:25.869 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:27.144+0000 7f61dcae3700 1 --2- 172.21.15.40:0/21794375 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f61d80fa8f0 0x7f61d80fad10 secure :-1 s=READY pgs=14 cs=0 l=1 rev1=1 rx=0x7f61c800b5f0 tx=0x7f61c8006ee0).stop 2023-11-17T05:21:25.869 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:27.144+0000 7f61dcae3700 1 -- 172.21.15.40:0/21794375 shutdown_connections 2023-11-17T05:21:25.869 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:27.144+0000 7f61dcae3700 1 --2- 172.21.15.40:0/21794375 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f61d80fa8f0 0x7f61d80fad10 unknown :-1 s=CLOSED pgs=14 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:21:25.869 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:27.144+0000 7f61dcae3700 1 -- 172.21.15.40:0/21794375 >> 172.21.15.40:0/21794375 conn(0x7f61d80f6130 msgr2=0x7f61d80f8550 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:21:25.869 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:27.144+0000 7f61dcae3700 1 -- 172.21.15.40:0/21794375 shutdown_connections 2023-11-17T05:21:25.869 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:27.144+0000 7f61dcae3700 1 -- 172.21.15.40:0/21794375 wait complete. 2023-11-17T05:21:25.869 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:27.145+0000 7f61dcae3700 1 Processor -- start 2023-11-17T05:21:25.869 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:27.145+0000 7f61dcae3700 1 -- start start 2023-11-17T05:21:25.869 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:27.145+0000 7f61dcae3700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f61d80fa8f0 0x7f61d8100b80 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:21:25.869 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:27.145+0000 7f61dcae3700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f61d80ff000 con 0x7f61d80fa8f0 2023-11-17T05:21:25.870 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:27.145+0000 7f61d77fe700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f61d80fa8f0 0x7f61d8100b80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:21:25.870 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:27.145+0000 7f61d77fe700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f61d80fa8f0 0x7f61d8100b80 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:54322/0 (socket says 172.21.15.40:54322) 2023-11-17T05:21:25.870 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:27.145+0000 7f61d77fe700 1 -- 172.21.15.40:0/2122377540 learned_addr learned my addr 172.21.15.40:0/2122377540 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-11-17T05:21:25.870 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:27.145+0000 7f61d77fe700 1 -- 172.21.15.40:0/2122377540 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f61c800b200 con 0x7f61d80fa8f0 2023-11-17T05:21:25.870 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:27.146+0000 7f61d77fe700 1 --2- 172.21.15.40:0/2122377540 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f61d80fa8f0 0x7f61d8100b80 secure :-1 s=READY pgs=15 cs=0 l=1 rev1=1 rx=0x7f61c800b1d0 tx=0x7f61c8006ee0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:21:25.870 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:27.146+0000 7f61bffff700 1 -- 172.21.15.40:0/2122377540 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f61c801b410 con 0x7f61d80fa8f0 2023-11-17T05:21:25.870 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:27.146+0000 7f61bffff700 1 -- 172.21.15.40:0/2122377540 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(25 keys) v1 ==== 1010+0+0 (secure 0 0 0) 0x7f61c801b9e0 con 0x7f61d80fa8f0 2023-11-17T05:21:25.870 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:27.146+0000 7f61dcae3700 1 -- 172.21.15.40:0/2122377540 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f61d80ff1a0 con 0x7f61d80fa8f0 2023-11-17T05:21:25.870 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:27.146+0000 7f61bffff700 1 -- 172.21.15.40:0/2122377540 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f61c801a840 con 0x7f61d80fa8f0 2023-11-17T05:21:25.870 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:27.146+0000 7f61dcae3700 1 -- 172.21.15.40:0/2122377540 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f61d80ff5b0 con 0x7f61d80fa8f0 2023-11-17T05:21:25.870 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:27.147+0000 7f61bffff700 1 -- 172.21.15.40:0/2122377540 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 2) v1 ==== 35959+0+0 (secure 0 0 0) 0x7f61c800f070 con 0x7f61d80fa8f0 2023-11-17T05:21:25.870 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:27.147+0000 7f61bffff700 1 -- 172.21.15.40:0/2122377540 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(1..1 src has 1..1) v4 ==== 754+0+0 (secure 0 0 0) 0x7f61c8040ff0 con 0x7f61d80fa8f0 2023-11-17T05:21:25.870 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:27.147+0000 7f61dcae3700 1 -- 172.21.15.40:0/2122377540 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f61d8059100 con 0x7f61d80fa8f0 2023-11-17T05:21:25.871 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:27.150+0000 7f61bffff700 1 -- 172.21.15.40:0/2122377540 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+72097 (secure 0 0 0) 0x7f61c8016020 con 0x7f61d80fa8f0 2023-11-17T05:21:25.871 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:27.221+0000 7f61dcae3700 1 -- 172.21.15.40:0/2122377540 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "status", "format": "json-pretty"} v 0) v1 -- 0x7f61d8045ad0 con 0x7f61d80fa8f0 2023-11-17T05:21:25.871 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:27.222+0000 7f61bffff700 1 -- 172.21.15.40:0/2122377540 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mon_command_ack([{"prefix": "status", "format": "json-pretty"}]=0 v0) v1 ==== 79+0+1230 (secure 0 0 0) 0x7f61c80164e0 con 0x7f61d80fa8f0 2023-11-17T05:21:25.871 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:25.871 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: { 2023-11-17T05:21:25.871 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: "fsid": "ed6470e2-8508-11ee-8dbe-212e2dc638e7", 2023-11-17T05:21:25.871 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: "health": { 2023-11-17T05:21:25.871 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: "status": "HEALTH_OK", 2023-11-17T05:21:25.871 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: "checks": {}, 2023-11-17T05:21:25.871 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: "mutes": [] 2023-11-17T05:21:25.871 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: }, 2023-11-17T05:21:25.871 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: "election_epoch": 5, 2023-11-17T05:21:25.871 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: "quorum": [ 2023-11-17T05:21:25.871 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 0 2023-11-17T05:21:25.871 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: ], 2023-11-17T05:21:25.872 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: "quorum_names": [ 2023-11-17T05:21:25.872 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: "smithi040" 2023-11-17T05:21:25.872 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: ], 2023-11-17T05:21:25.872 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: "quorum_age": 7, 2023-11-17T05:21:25.872 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: "monmap": { 2023-11-17T05:21:25.872 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: "epoch": 1, 2023-11-17T05:21:25.872 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: "min_mon_release_name": "pacific", 2023-11-17T05:21:25.872 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: "num_mons": 1 2023-11-17T05:21:25.872 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: }, 2023-11-17T05:21:25.872 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: "osdmap": { 2023-11-17T05:21:25.872 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: "epoch": 1, 2023-11-17T05:21:25.872 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: "num_osds": 0, 2023-11-17T05:21:25.872 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: "num_up_osds": 0, 2023-11-17T05:21:25.873 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: "osd_up_since": 0, 2023-11-17T05:21:25.873 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: "num_in_osds": 0, 2023-11-17T05:21:25.873 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: "osd_in_since": 0, 2023-11-17T05:21:25.873 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: "num_remapped_pgs": 0 2023-11-17T05:21:25.873 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: }, 2023-11-17T05:21:25.873 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: "pgmap": { 2023-11-17T05:21:25.873 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: "pgs_by_state": [], 2023-11-17T05:21:25.873 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: "num_pgs": 0, 2023-11-17T05:21:25.873 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: "num_pools": 0, 2023-11-17T05:21:25.873 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: "num_objects": 0, 2023-11-17T05:21:25.873 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: "data_bytes": 0, 2023-11-17T05:21:25.873 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: "bytes_used": 0, 2023-11-17T05:21:25.873 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: "bytes_avail": 0, 2023-11-17T05:21:25.873 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: "bytes_total": 0 2023-11-17T05:21:25.874 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: }, 2023-11-17T05:21:25.874 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: "fsmap": { 2023-11-17T05:21:25.874 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: "epoch": 1, 2023-11-17T05:21:25.874 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: "by_rank": [], 2023-11-17T05:21:25.874 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: "up:standby": 0 2023-11-17T05:21:25.874 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: }, 2023-11-17T05:21:25.874 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: "mgrmap": { 2023-11-17T05:21:25.874 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: "available": false, 2023-11-17T05:21:25.874 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: "num_standbys": 0, 2023-11-17T05:21:25.874 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: "modules": [ 2023-11-17T05:21:25.874 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: "iostat", 2023-11-17T05:21:25.874 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: "restful" 2023-11-17T05:21:25.874 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: ], 2023-11-17T05:21:25.875 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: "services": {} 2023-11-17T05:21:25.875 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: }, 2023-11-17T05:21:25.875 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: "servicemap": { 2023-11-17T05:21:25.875 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: "epoch": 1, 2023-11-17T05:21:25.875 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: "modified": "2023-11-17T05:20:15.855428+0000", 2023-11-17T05:21:25.875 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: "services": {} 2023-11-17T05:21:25.875 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: }, 2023-11-17T05:21:25.875 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: "progress_events": {} 2023-11-17T05:21:25.875 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: } 2023-11-17T05:21:25.875 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:27.222+0000 7f61dcae3700 1 -- 172.21.15.40:0/2122377540 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f61d80fa8f0 msgr2=0x7f61d8100b80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:21:25.875 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:27.222+0000 7f61dcae3700 1 --2- 172.21.15.40:0/2122377540 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f61d80fa8f0 0x7f61d8100b80 secure :-1 s=READY pgs=15 cs=0 l=1 rev1=1 rx=0x7f61c800b1d0 tx=0x7f61c8006ee0).stop 2023-11-17T05:21:25.875 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:27.222+0000 7f61dcae3700 1 -- 172.21.15.40:0/2122377540 shutdown_connections 2023-11-17T05:21:25.875 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:27.222+0000 7f61dcae3700 1 --2- 172.21.15.40:0/2122377540 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f61d80fa8f0 0x7f61d8100b80 unknown :-1 s=CLOSED pgs=15 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:21:25.876 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:27.222+0000 7f61dcae3700 1 -- 172.21.15.40:0/2122377540 >> 172.21.15.40:0/2122377540 conn(0x7f61d80f6130 msgr2=0x7f61d806f830 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:21:25.876 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:27.223+0000 7f61dcae3700 1 -- 172.21.15.40:0/2122377540 shutdown_connections 2023-11-17T05:21:25.876 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:27.223+0000 7f61dcae3700 1 -- 172.21.15.40:0/2122377540 wait complete. 2023-11-17T05:21:25.876 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:27 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:20:27.222565+0000 mon.smithi040 (mon.0) 22 : audit [DBG] from='client.? 172.21.15.40:0/2122377540' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2023-11-17T05:21:25.876 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:27 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:20:27.416938+0000 mon.smithi040 (mon.0) 23 : cluster [DBG] mgrmap e3: smithi040.ogcprl(active, since 1.0064s) 2023-11-17T05:21:25.876 INFO:teuthology.orchestra.run.smithi040.stderr:mgr not available, waiting (3/15)... 2023-11-17T05:21:25.877 INFO:teuthology.orchestra.run.smithi040.stderr:Running command: /bin/podman run --rm --ipc=host --no-hosts --net=host --entrypoint /usr/bin/ceph --init -e CONTAINER_IMAGE=docker.io/ceph/ceph:v16.2.4 -e NODE_NAME=smithi040 -e CEPH_USE_RANDOM_NONCE=1 -v /var/log/ceph/ed6470e2-8508-11ee-8dbe-212e2dc638e7:/var/log/ceph:z -v /tmp/ceph-tmpzx79_7_h:/etc/ceph/ceph.client.admin.keyring:z -v /tmp/ceph-tmpru9tjirw:/etc/ceph/ceph.conf:z docker.io/ceph/ceph:v16.2.4 status -f json-pretty 2023-11-17T05:21:25.877 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:29 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:20:28.468599+0000 mon.smithi040 (mon.0) 24 : cluster [DBG] mgrmap e4: smithi040.ogcprl(active, since 2s) 2023-11-17T05:21:25.886 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:30.078+0000 7ff90ee23700 1 Processor -- start 2023-11-17T05:21:25.886 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:30.078+0000 7ff90ee23700 1 -- start start 2023-11-17T05:21:25.886 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:30.078+0000 7ff90ee23700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7ff90806fc30 0x7ff9080fe650 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 rx=0 tx=0).connect 2023-11-17T05:21:25.886 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:30.078+0000 7ff90ee23700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7ff9080feb40 con 0x7ff90806fc30 2023-11-17T05:21:25.886 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:30.079+0000 7ff90de21700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7ff90806fc30 0x7ff9080fe650 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:21:25.886 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:30.079+0000 7ff90de21700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7ff90806fc30 0x7ff9080fe650 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:54332/0 (socket says 172.21.15.40:54332) 2023-11-17T05:21:25.886 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:30.079+0000 7ff90de21700 1 -- 172.21.15.40:0/3944631192 learned_addr learned my addr 172.21.15.40:0/3944631192 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-11-17T05:21:25.886 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:30.079+0000 7ff90de21700 1 -- 172.21.15.40:0/3944631192 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7ff9080fec80 con 0x7ff90806fc30 2023-11-17T05:21:25.887 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:30.079+0000 7ff90de21700 1 --2- 172.21.15.40:0/3944631192 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7ff90806fc30 0x7ff9080fe650 secure :-1 s=READY pgs=16 cs=0 l=1 rev1=1 rx=0x7ff90400b5f0 tx=0x7ff904006ee0).ready entity=mon.0 client_cookie=ce2341c1a0da6a5e server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:21:25.887 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:30.079+0000 7ff90ce1f700 1 -- 172.21.15.40:0/3944631192 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7ff90400d600 con 0x7ff90806fc30 2023-11-17T05:21:25.887 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:30.079+0000 7ff90ce1f700 1 -- 172.21.15.40:0/3944631192 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(25 keys) v1 ==== 1010+0+0 (secure 0 0 0) 0x7ff90400dbd0 con 0x7ff90806fc30 2023-11-17T05:21:25.887 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:30.079+0000 7ff90ce1f700 1 -- 172.21.15.40:0/3944631192 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7ff904011c00 con 0x7ff90806fc30 2023-11-17T05:21:25.887 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:30.080+0000 7ff90ee23700 1 -- 172.21.15.40:0/3944631192 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7ff90806fc30 msgr2=0x7ff9080fe650 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:21:25.887 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:30.080+0000 7ff90ee23700 1 --2- 172.21.15.40:0/3944631192 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7ff90806fc30 0x7ff9080fe650 secure :-1 s=READY pgs=16 cs=0 l=1 rev1=1 rx=0x7ff90400b5f0 tx=0x7ff904006ee0).stop 2023-11-17T05:21:25.887 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:30.080+0000 7ff90ee23700 1 -- 172.21.15.40:0/3944631192 shutdown_connections 2023-11-17T05:21:25.887 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:30.080+0000 7ff90ee23700 1 --2- 172.21.15.40:0/3944631192 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7ff90806fc30 0x7ff9080fe650 unknown :-1 s=CLOSED pgs=16 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:21:25.887 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:30.080+0000 7ff90ee23700 1 -- 172.21.15.40:0/3944631192 >> 172.21.15.40:0/3944631192 conn(0x7ff9080f6130 msgr2=0x7ff9080f8550 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:21:25.887 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:30.080+0000 7ff90ee23700 1 -- 172.21.15.40:0/3944631192 shutdown_connections 2023-11-17T05:21:25.887 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:30.080+0000 7ff90ee23700 1 -- 172.21.15.40:0/3944631192 wait complete. 2023-11-17T05:21:25.887 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:30.081+0000 7ff90ee23700 1 Processor -- start 2023-11-17T05:21:25.887 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:30.081+0000 7ff90ee23700 1 -- start start 2023-11-17T05:21:25.888 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:30.081+0000 7ff90ee23700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7ff90806fc30 0x7ff90810a7e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:21:25.888 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:30.081+0000 7ff90ee23700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7ff90810acd0 con 0x7ff90806fc30 2023-11-17T05:21:25.888 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:30.081+0000 7ff90de21700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7ff90806fc30 0x7ff90810a7e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:21:25.888 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:30.081+0000 7ff90de21700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7ff90806fc30 0x7ff90810a7e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:54340/0 (socket says 172.21.15.40:54340) 2023-11-17T05:21:25.888 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:30.081+0000 7ff90de21700 1 -- 172.21.15.40:0/1036162412 learned_addr learned my addr 172.21.15.40:0/1036162412 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-11-17T05:21:25.888 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:30.082+0000 7ff90de21700 1 -- 172.21.15.40:0/1036162412 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7ff90400b200 con 0x7ff90806fc30 2023-11-17T05:21:25.888 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:30.082+0000 7ff90de21700 1 --2- 172.21.15.40:0/1036162412 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7ff90806fc30 0x7ff90810a7e0 secure :-1 s=READY pgs=17 cs=0 l=1 rev1=1 rx=0x7ff90400e040 tx=0x7ff904006ee0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:21:25.888 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:30.082+0000 7ff8f67fc700 1 -- 172.21.15.40:0/1036162412 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7ff90400d9f0 con 0x7ff90806fc30 2023-11-17T05:21:25.888 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:30.082+0000 7ff8f67fc700 1 -- 172.21.15.40:0/1036162412 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(25 keys) v1 ==== 1010+0+0 (secure 0 0 0) 0x7ff90401a9f0 con 0x7ff90806fc30 2023-11-17T05:21:25.888 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:30.082+0000 7ff90ee23700 1 -- 172.21.15.40:0/1036162412 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7ff90810aed0 con 0x7ff90806fc30 2023-11-17T05:21:25.888 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:30.082+0000 7ff8f67fc700 1 -- 172.21.15.40:0/1036162412 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7ff90401b8c0 con 0x7ff90806fc30 2023-11-17T05:21:25.888 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:30.082+0000 7ff90ee23700 1 -- 172.21.15.40:0/1036162412 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7ff90810b2e0 con 0x7ff90806fc30 2023-11-17T05:21:25.888 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:30.083+0000 7ff8f67fc700 1 -- 172.21.15.40:0/1036162412 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 4) v1 ==== 36109+0+0 (secure 0 0 0) 0x7ff90400f070 con 0x7ff90806fc30 2023-11-17T05:21:25.888 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:30.083+0000 7ff8f67fc700 1 --2- 172.21.15.40:0/1036162412 >> [v2:172.21.15.40:6800/74978488,v1:172.21.15.40:6801/74978488] conn(0x7ff8f002fd50 0x7ff8f0032210 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:21:25.889 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:30.083+0000 7ff8f67fc700 1 -- 172.21.15.40:0/1036162412 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(1..1 src has 1..1) v4 ==== 754+0+0 (secure 0 0 0) 0x7ff904041190 con 0x7ff90806fc30 2023-11-17T05:21:25.889 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:30.083+0000 7ff90d620700 1 --2- 172.21.15.40:0/1036162412 >> [v2:172.21.15.40:6800/74978488,v1:172.21.15.40:6801/74978488] conn(0x7ff8f002fd50 0x7ff8f0032210 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:21:25.889 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:30.083+0000 7ff90d620700 1 --2- 172.21.15.40:0/1036162412 >> [v2:172.21.15.40:6800/74978488,v1:172.21.15.40:6801/74978488] conn(0x7ff8f002fd50 0x7ff8f0032210 secure :-1 s=READY pgs=5 cs=0 l=1 rev1=1 rx=0x7ff8f80096a0 tx=0x7ff8f8006b40).ready entity=mgr.14100 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:21:25.889 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:30.084+0000 7ff90ee23700 1 -- 172.21.15.40:0/1036162412 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7ff908059100 con 0x7ff90806fc30 2023-11-17T05:21:25.889 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:30.089+0000 7ff8f67fc700 1 -- 172.21.15.40:0/1036162412 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+152212 (secure 0 0 0) 0x7ff904016070 con 0x7ff90806fc30 2023-11-17T05:21:25.889 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:30.316+0000 7ff90ee23700 1 -- 172.21.15.40:0/1036162412 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "status", "format": "json-pretty"} v 0) v1 -- 0x7ff908045ad0 con 0x7ff90806fc30 2023-11-17T05:21:25.889 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:30.316+0000 7ff8f67fc700 1 -- 172.21.15.40:0/1036162412 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mon_command_ack([{"prefix": "status", "format": "json-pretty"}]=0 v0) v1 ==== 79+0+1230 (secure 0 0 0) 0x7ff904022900 con 0x7ff90806fc30 2023-11-17T05:21:25.889 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:25.889 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: { 2023-11-17T05:21:25.889 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: "fsid": "ed6470e2-8508-11ee-8dbe-212e2dc638e7", 2023-11-17T05:21:25.889 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: "health": { 2023-11-17T05:21:25.889 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: "status": "HEALTH_OK", 2023-11-17T05:21:25.889 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: "checks": {}, 2023-11-17T05:21:25.890 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: "mutes": [] 2023-11-17T05:21:25.894 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: }, 2023-11-17T05:21:25.895 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: "election_epoch": 5, 2023-11-17T05:21:25.895 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: "quorum": [ 2023-11-17T05:21:25.895 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 0 2023-11-17T05:21:25.895 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: ], 2023-11-17T05:21:25.895 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: "quorum_names": [ 2023-11-17T05:21:25.895 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: "smithi040" 2023-11-17T05:21:25.895 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: ], 2023-11-17T05:21:25.895 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: "quorum_age": 10, 2023-11-17T05:21:25.895 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: "monmap": { 2023-11-17T05:21:25.895 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: "epoch": 1, 2023-11-17T05:21:25.895 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: "min_mon_release_name": "pacific", 2023-11-17T05:21:25.895 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: "num_mons": 1 2023-11-17T05:21:25.895 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: }, 2023-11-17T05:21:25.895 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: "osdmap": { 2023-11-17T05:21:25.896 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: "epoch": 1, 2023-11-17T05:21:25.896 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: "num_osds": 0, 2023-11-17T05:21:25.896 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: "num_up_osds": 0, 2023-11-17T05:21:25.896 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: "osd_up_since": 0, 2023-11-17T05:21:25.896 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: "num_in_osds": 0, 2023-11-17T05:21:25.896 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: "osd_in_since": 0, 2023-11-17T05:21:25.896 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: "num_remapped_pgs": 0 2023-11-17T05:21:25.896 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: }, 2023-11-17T05:21:25.896 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: "pgmap": { 2023-11-17T05:21:25.896 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: "pgs_by_state": [], 2023-11-17T05:21:25.896 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: "num_pgs": 0, 2023-11-17T05:21:25.896 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: "num_pools": 0, 2023-11-17T05:21:25.896 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: "num_objects": 0, 2023-11-17T05:21:25.896 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: "data_bytes": 0, 2023-11-17T05:21:25.897 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: "bytes_used": 0, 2023-11-17T05:21:25.897 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: "bytes_avail": 0, 2023-11-17T05:21:25.897 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: "bytes_total": 0 2023-11-17T05:21:25.897 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: }, 2023-11-17T05:21:25.897 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: "fsmap": { 2023-11-17T05:21:25.897 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: "epoch": 1, 2023-11-17T05:21:25.897 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: "by_rank": [], 2023-11-17T05:21:25.897 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: "up:standby": 0 2023-11-17T05:21:25.897 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: }, 2023-11-17T05:21:25.897 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: "mgrmap": { 2023-11-17T05:21:25.897 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: "available": true, 2023-11-17T05:21:25.897 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: "num_standbys": 0, 2023-11-17T05:21:25.897 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: "modules": [ 2023-11-17T05:21:25.898 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: "iostat", 2023-11-17T05:21:25.898 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: "restful" 2023-11-17T05:21:25.898 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: ], 2023-11-17T05:21:25.965 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: "services": {} 2023-11-17T05:21:25.965 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: }, 2023-11-17T05:21:25.965 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: "servicemap": { 2023-11-17T05:21:25.965 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: "epoch": 1, 2023-11-17T05:21:25.965 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: "modified": "2023-11-17T05:20:15.855428+0000", 2023-11-17T05:21:25.965 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: "services": {} 2023-11-17T05:21:25.965 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: }, 2023-11-17T05:21:25.965 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: "progress_events": {} 2023-11-17T05:21:25.965 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: } 2023-11-17T05:21:25.965 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:30.318+0000 7ff90ee23700 1 -- 172.21.15.40:0/1036162412 >> [v2:172.21.15.40:6800/74978488,v1:172.21.15.40:6801/74978488] conn(0x7ff8f002fd50 msgr2=0x7ff8f0032210 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:21:25.965 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:30.318+0000 7ff90ee23700 1 --2- 172.21.15.40:0/1036162412 >> [v2:172.21.15.40:6800/74978488,v1:172.21.15.40:6801/74978488] conn(0x7ff8f002fd50 0x7ff8f0032210 secure :-1 s=READY pgs=5 cs=0 l=1 rev1=1 rx=0x7ff8f80096a0 tx=0x7ff8f8006b40).stop 2023-11-17T05:21:25.965 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:30.318+0000 7ff90ee23700 1 -- 172.21.15.40:0/1036162412 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7ff90806fc30 msgr2=0x7ff90810a7e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:21:25.966 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:30.318+0000 7ff90ee23700 1 --2- 172.21.15.40:0/1036162412 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7ff90806fc30 0x7ff90810a7e0 secure :-1 s=READY pgs=17 cs=0 l=1 rev1=1 rx=0x7ff90400e040 tx=0x7ff904006ee0).stop 2023-11-17T05:21:25.966 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:30.318+0000 7ff90ee23700 1 -- 172.21.15.40:0/1036162412 shutdown_connections 2023-11-17T05:21:25.966 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:30.318+0000 7ff90ee23700 1 --2- 172.21.15.40:0/1036162412 >> [v2:172.21.15.40:6800/74978488,v1:172.21.15.40:6801/74978488] conn(0x7ff8f002fd50 0x7ff8f0032210 unknown :-1 s=CLOSED pgs=5 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:21:25.966 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:30.318+0000 7ff90ee23700 1 --2- 172.21.15.40:0/1036162412 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7ff90806fc30 0x7ff90810a7e0 unknown :-1 s=CLOSED pgs=17 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:21:25.967 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:30.318+0000 7ff90ee23700 1 -- 172.21.15.40:0/1036162412 >> 172.21.15.40:0/1036162412 conn(0x7ff9080f6130 msgr2=0x7ff9080f7c60 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:21:26.153 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:30.319+0000 7ff90ee23700 1 -- 172.21.15.40:0/1036162412 shutdown_connections 2023-11-17T05:21:26.153 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:30.319+0000 7ff90ee23700 1 -- 172.21.15.40:0/1036162412 wait complete. 2023-11-17T05:21:26.153 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:30 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:20:30.317069+0000 mon.smithi040 (mon.0) 25 : audit [DBG] from='client.? 172.21.15.40:0/1036162412' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2023-11-17T05:21:26.166 INFO:teuthology.orchestra.run.smithi040.stderr:mgr is available 2023-11-17T05:21:26.168 INFO:teuthology.orchestra.run.smithi040.stderr:Enabling cephadm module... 2023-11-17T05:21:26.168 INFO:teuthology.orchestra.run.smithi040.stderr:Running command: /bin/podman run --rm --ipc=host --no-hosts --net=host --entrypoint /usr/bin/ceph --init -e CONTAINER_IMAGE=docker.io/ceph/ceph:v16.2.4 -e NODE_NAME=smithi040 -e CEPH_USE_RANDOM_NONCE=1 -v /var/log/ceph/ed6470e2-8508-11ee-8dbe-212e2dc638e7:/var/log/ceph:z -v /tmp/ceph-tmpzx79_7_h:/etc/ceph/ceph.client.admin.keyring:z -v /tmp/ceph-tmpru9tjirw:/etc/ceph/ceph.conf:z docker.io/ceph/ceph:v16.2.4 mgr module enable cephadm 2023-11-17T05:21:26.168 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:31.187+0000 7fd2a0ca8700 1 Processor -- start 2023-11-17T05:21:26.168 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:31.187+0000 7fd2a0ca8700 1 -- start start 2023-11-17T05:21:26.168 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:31.187+0000 7fd2a0ca8700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fd29c0f9f60 0x7fd29c0fa380 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 rx=0 tx=0).connect 2023-11-17T05:21:26.168 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:31.187+0000 7fd2a0ca8700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fd29c0fa900 con 0x7fd29c0f9f60 2023-11-17T05:21:26.168 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:31.187+0000 7fd29a59c700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fd29c0f9f60 0x7fd29c0fa380 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:21:26.169 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:31.187+0000 7fd29a59c700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fd29c0f9f60 0x7fd29c0fa380 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:54350/0 (socket says 172.21.15.40:54350) 2023-11-17T05:21:26.169 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:31.187+0000 7fd29a59c700 1 -- 172.21.15.40:0/1546472284 learned_addr learned my addr 172.21.15.40:0/1546472284 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-11-17T05:21:26.169 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:31.188+0000 7fd29a59c700 1 -- 172.21.15.40:0/1546472284 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fd29c0faa40 con 0x7fd29c0f9f60 2023-11-17T05:21:26.169 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:31.188+0000 7fd29a59c700 1 --2- 172.21.15.40:0/1546472284 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fd29c0f9f60 0x7fd29c0fa380 secure :-1 s=READY pgs=18 cs=0 l=1 rev1=1 rx=0x7fd28c00b5f0 tx=0x7fd28c006ee0).ready entity=mon.0 client_cookie=5df3210a12315381 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:21:26.169 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:31.188+0000 7fd29959a700 1 -- 172.21.15.40:0/1546472284 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7fd28c00d600 con 0x7fd29c0f9f60 2023-11-17T05:21:26.169 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:31.188+0000 7fd29959a700 1 -- 172.21.15.40:0/1546472284 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(25 keys) v1 ==== 1010+0+0 (secure 0 0 0) 0x7fd28c00dbd0 con 0x7fd29c0f9f60 2023-11-17T05:21:26.169 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:31.188+0000 7fd29959a700 1 -- 172.21.15.40:0/1546472284 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7fd28c011c00 con 0x7fd29c0f9f60 2023-11-17T05:21:26.169 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:31.188+0000 7fd2a0ca8700 1 -- 172.21.15.40:0/1546472284 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fd29c0f9f60 msgr2=0x7fd29c0fa380 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:21:26.169 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:31.188+0000 7fd2a0ca8700 1 --2- 172.21.15.40:0/1546472284 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fd29c0f9f60 0x7fd29c0fa380 secure :-1 s=READY pgs=18 cs=0 l=1 rev1=1 rx=0x7fd28c00b5f0 tx=0x7fd28c006ee0).stop 2023-11-17T05:21:26.169 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:31.189+0000 7fd2a0ca8700 1 -- 172.21.15.40:0/1546472284 shutdown_connections 2023-11-17T05:21:26.169 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:31.189+0000 7fd2a0ca8700 1 --2- 172.21.15.40:0/1546472284 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fd29c0f9f60 0x7fd29c0fa380 unknown :-1 s=CLOSED pgs=18 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:21:26.170 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:31.189+0000 7fd2a0ca8700 1 -- 172.21.15.40:0/1546472284 >> 172.21.15.40:0/1546472284 conn(0x7fd29c0f59e0 msgr2=0x7fd29c0f7e00 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:21:26.170 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:31.189+0000 7fd2a0ca8700 1 -- 172.21.15.40:0/1546472284 shutdown_connections 2023-11-17T05:21:26.170 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:31.189+0000 7fd2a0ca8700 1 -- 172.21.15.40:0/1546472284 wait complete. 2023-11-17T05:21:26.170 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:31.189+0000 7fd2a0ca8700 1 Processor -- start 2023-11-17T05:21:26.178 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:31.189+0000 7fd2a0ca8700 1 -- start start 2023-11-17T05:21:26.181 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:31.189+0000 7fd2a0ca8700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fd29c0f9f60 0x7fd29c10cbd0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:21:26.181 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:31.190+0000 7fd2a0ca8700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fd29c10d0c0 con 0x7fd29c0f9f60 2023-11-17T05:21:26.181 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:31.190+0000 7fd29a59c700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fd29c0f9f60 0x7fd29c10cbd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:21:26.181 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:31.190+0000 7fd29a59c700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fd29c0f9f60 0x7fd29c10cbd0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:54364/0 (socket says 172.21.15.40:54364) 2023-11-17T05:21:26.181 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:31.190+0000 7fd29a59c700 1 -- 172.21.15.40:0/1801812697 learned_addr learned my addr 172.21.15.40:0/1801812697 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-11-17T05:21:26.181 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:31.190+0000 7fd29a59c700 1 -- 172.21.15.40:0/1801812697 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fd28c00b200 con 0x7fd29c0f9f60 2023-11-17T05:21:26.181 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:31.190+0000 7fd29a59c700 1 --2- 172.21.15.40:0/1801812697 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fd29c0f9f60 0x7fd29c10cbd0 secure :-1 s=READY pgs=19 cs=0 l=1 rev1=1 rx=0x7fd28c00e040 tx=0x7fd28c006ee0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:21:26.181 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:31.190+0000 7fd292ffd700 1 -- 172.21.15.40:0/1801812697 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7fd28c01b410 con 0x7fd29c0f9f60 2023-11-17T05:21:26.181 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:31.190+0000 7fd292ffd700 1 -- 172.21.15.40:0/1801812697 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(25 keys) v1 ==== 1010+0+0 (secure 0 0 0) 0x7fd28c01b9e0 con 0x7fd29c0f9f60 2023-11-17T05:21:26.181 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:31.190+0000 7fd292ffd700 1 -- 172.21.15.40:0/1801812697 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7fd28c01a770 con 0x7fd29c0f9f60 2023-11-17T05:21:26.181 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:31.190+0000 7fd2a0ca8700 1 -- 172.21.15.40:0/1801812697 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fd29c10d2c0 con 0x7fd29c0f9f60 2023-11-17T05:21:26.182 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:31.190+0000 7fd2a0ca8700 1 -- 172.21.15.40:0/1801812697 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fd29c10d6d0 con 0x7fd29c0f9f60 2023-11-17T05:21:26.182 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:31.191+0000 7fd292ffd700 1 -- 172.21.15.40:0/1801812697 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 4) v1 ==== 36109+0+0 (secure 0 0 0) 0x7fd28c00f070 con 0x7fd29c0f9f60 2023-11-17T05:21:26.182 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:31.191+0000 7fd292ffd700 1 --2- 172.21.15.40:0/1801812697 >> [v2:172.21.15.40:6800/74978488,v1:172.21.15.40:6801/74978488] conn(0x7fd27c02fd50 0x7fd27c032210 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:21:26.182 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:31.191+0000 7fd292ffd700 1 -- 172.21.15.40:0/1801812697 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(1..1 src has 1..1) v4 ==== 754+0+0 (secure 0 0 0) 0x7fd28c040f00 con 0x7fd29c0f9f60 2023-11-17T05:21:26.182 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:31.191+0000 7fd2a0ca8700 1 -- 172.21.15.40:0/1801812697 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fd29c045ad0 con 0x7fd29c0f9f60 2023-11-17T05:21:26.182 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:31.192+0000 7fd299d9b700 1 --2- 172.21.15.40:0/1801812697 >> [v2:172.21.15.40:6800/74978488,v1:172.21.15.40:6801/74978488] conn(0x7fd27c02fd50 0x7fd27c032210 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:21:26.182 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:31.192+0000 7fd299d9b700 1 --2- 172.21.15.40:0/1801812697 >> [v2:172.21.15.40:6800/74978488,v1:172.21.15.40:6801/74978488] conn(0x7fd27c02fd50 0x7fd27c032210 secure :-1 s=READY pgs=6 cs=0 l=1 rev1=1 rx=0x7fd2840096a0 tx=0x7fd284006b40).ready entity=mgr.14100 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:21:26.182 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:31.196+0000 7fd292ffd700 1 -- 172.21.15.40:0/1801812697 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+152212 (secure 0 0 0) 0x7fd28c016080 con 0x7fd29c0f9f60 2023-11-17T05:21:26.182 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:31.402+0000 7fd2a0ca8700 1 -- 172.21.15.40:0/1801812697 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "mgr module enable", "module": "cephadm"} v 0) v1 -- 0x7fd29c0ffbc0 con 0x7fd29c0f9f60 2023-11-17T05:21:26.182 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:31.471+0000 7fd292ffd700 1 -- 172.21.15.40:0/1801812697 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mgrmap(e 5) v1 ==== 36120+0+0 (secure 0 0 0) 0x7fd28c01bb50 con 0x7fd29c0f9f60 2023-11-17T05:21:26.182 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:31.472+0000 7fd292ffd700 1 -- 172.21.15.40:0/1801812697 <== mon.0 v2:172.21.15.40:3300/0 8 ==== mon_command_ack([{"prefix": "mgr module enable", "module": "cephadm"}]=0 v5) v1 ==== 86+0+0 (secure 0 0 0) 0x7fd28c023600 con 0x7fd29c0f9f60 2023-11-17T05:21:26.183 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:31.474+0000 7fd2a0ca8700 1 -- 172.21.15.40:0/1801812697 >> [v2:172.21.15.40:6800/74978488,v1:172.21.15.40:6801/74978488] conn(0x7fd27c02fd50 msgr2=0x7fd27c032210 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:21:26.183 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:31.474+0000 7fd2a0ca8700 1 --2- 172.21.15.40:0/1801812697 >> [v2:172.21.15.40:6800/74978488,v1:172.21.15.40:6801/74978488] conn(0x7fd27c02fd50 0x7fd27c032210 secure :-1 s=READY pgs=6 cs=0 l=1 rev1=1 rx=0x7fd2840096a0 tx=0x7fd284006b40).stop 2023-11-17T05:21:26.183 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:31.474+0000 7fd2a0ca8700 1 -- 172.21.15.40:0/1801812697 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fd29c0f9f60 msgr2=0x7fd29c10cbd0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:21:26.183 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:31.474+0000 7fd2a0ca8700 1 --2- 172.21.15.40:0/1801812697 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fd29c0f9f60 0x7fd29c10cbd0 secure :-1 s=READY pgs=19 cs=0 l=1 rev1=1 rx=0x7fd28c00e040 tx=0x7fd28c006ee0).stop 2023-11-17T05:21:26.183 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:31.474+0000 7fd2a0ca8700 1 -- 172.21.15.40:0/1801812697 shutdown_connections 2023-11-17T05:21:26.183 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:31.474+0000 7fd2a0ca8700 1 --2- 172.21.15.40:0/1801812697 >> [v2:172.21.15.40:6800/74978488,v1:172.21.15.40:6801/74978488] conn(0x7fd27c02fd50 0x7fd27c032210 unknown :-1 s=CLOSED pgs=6 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:21:26.183 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:31.474+0000 7fd2a0ca8700 1 --2- 172.21.15.40:0/1801812697 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fd29c0f9f60 0x7fd29c10cbd0 unknown :-1 s=CLOSED pgs=19 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:21:26.183 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:31.474+0000 7fd2a0ca8700 1 -- 172.21.15.40:0/1801812697 >> 172.21.15.40:0/1801812697 conn(0x7fd29c0f59e0 msgr2=0x7fd29c0f7530 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:21:26.183 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:31.475+0000 7fd2a0ca8700 1 -- 172.21.15.40:0/1801812697 shutdown_connections 2023-11-17T05:21:26.183 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:31.475+0000 7fd2a0ca8700 1 -- 172.21.15.40:0/1801812697 wait complete. 2023-11-17T05:21:26.184 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:31 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:20:31.403019+0000 mon.smithi040 (mon.0) 26 : audit [INF] from='client.? 172.21.15.40:0/1801812697' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "cephadm"}]: dispatch 2023-11-17T05:21:26.191 INFO:teuthology.orchestra.run.smithi040.stderr:Running command: /bin/podman run --rm --ipc=host --no-hosts --net=host --entrypoint /usr/bin/ceph --init -e CONTAINER_IMAGE=docker.io/ceph/ceph:v16.2.4 -e NODE_NAME=smithi040 -e CEPH_USE_RANDOM_NONCE=1 -v /var/log/ceph/ed6470e2-8508-11ee-8dbe-212e2dc638e7:/var/log/ceph:z -v /tmp/ceph-tmpzx79_7_h:/etc/ceph/ceph.client.admin.keyring:z -v /tmp/ceph-tmpru9tjirw:/etc/ceph/ceph.conf:z docker.io/ceph/ceph:v16.2.4 mgr stat 2023-11-17T05:21:26.191 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:32.449+0000 7f1d34e41700 1 Processor -- start 2023-11-17T05:21:26.191 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:32.449+0000 7f1d34e41700 1 -- start start 2023-11-17T05:21:26.194 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:32.450+0000 7f1d34e41700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f1d3006fbd0 0x7f1d30100c10 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 rx=0 tx=0).connect 2023-11-17T05:21:26.194 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:32.450+0000 7f1d34e41700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f1d300fc600 con 0x7f1d3006fbd0 2023-11-17T05:21:26.194 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:32.450+0000 7f1d2e59c700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f1d3006fbd0 0x7f1d30100c10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:21:26.195 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:32.450+0000 7f1d2e59c700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f1d3006fbd0 0x7f1d30100c10 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:54402/0 (socket says 172.21.15.40:54402) 2023-11-17T05:21:26.195 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:32.450+0000 7f1d2e59c700 1 -- 172.21.15.40:0/1679774716 learned_addr learned my addr 172.21.15.40:0/1679774716 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-11-17T05:21:26.195 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:32.450+0000 7f1d2e59c700 1 -- 172.21.15.40:0/1679774716 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f1d30101100 con 0x7f1d3006fbd0 2023-11-17T05:21:26.195 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:32.450+0000 7f1d2e59c700 1 --2- 172.21.15.40:0/1679774716 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f1d3006fbd0 0x7f1d30100c10 secure :-1 s=READY pgs=22 cs=0 l=1 rev1=1 rx=0x7f1d2000b5f0 tx=0x7f1d20006ee0).ready entity=mon.0 client_cookie=24e6975e2a1d8452 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:21:26.195 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:32.451+0000 7f1d2d59a700 1 -- 172.21.15.40:0/1679774716 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f1d2000d600 con 0x7f1d3006fbd0 2023-11-17T05:21:26.195 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:32.451+0000 7f1d2d59a700 1 -- 172.21.15.40:0/1679774716 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(25 keys) v1 ==== 1010+0+0 (secure 0 0 0) 0x7f1d2000dbd0 con 0x7f1d3006fbd0 2023-11-17T05:21:26.195 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:32.451+0000 7f1d2d59a700 1 -- 172.21.15.40:0/1679774716 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f1d20011c00 con 0x7f1d3006fbd0 2023-11-17T05:21:26.195 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:32.451+0000 7f1d34e41700 1 -- 172.21.15.40:0/1679774716 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f1d3006fbd0 msgr2=0x7f1d30100c10 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:21:26.195 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:32.451+0000 7f1d34e41700 1 --2- 172.21.15.40:0/1679774716 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f1d3006fbd0 0x7f1d30100c10 secure :-1 s=READY pgs=22 cs=0 l=1 rev1=1 rx=0x7f1d2000b5f0 tx=0x7f1d20006ee0).stop 2023-11-17T05:21:26.195 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:32.452+0000 7f1d34e41700 1 -- 172.21.15.40:0/1679774716 shutdown_connections 2023-11-17T05:21:26.195 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:32.452+0000 7f1d34e41700 1 --2- 172.21.15.40:0/1679774716 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f1d3006fbd0 0x7f1d30100c10 unknown :-1 s=CLOSED pgs=22 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:21:26.195 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:32.452+0000 7f1d34e41700 1 -- 172.21.15.40:0/1679774716 >> 172.21.15.40:0/1679774716 conn(0x7f1d300f6270 msgr2=0x7f1d300f8690 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:21:26.196 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:32.452+0000 7f1d34e41700 1 -- 172.21.15.40:0/1679774716 shutdown_connections 2023-11-17T05:21:26.196 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:32.452+0000 7f1d34e41700 1 -- 172.21.15.40:0/1679774716 wait complete. 2023-11-17T05:21:26.196 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:32.452+0000 7f1d34e41700 1 Processor -- start 2023-11-17T05:21:26.196 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:32.452+0000 7f1d34e41700 1 -- start start 2023-11-17T05:21:26.196 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:32.453+0000 7f1d34e41700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f1d3006fbd0 0x7f1d3010cac0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:21:26.196 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:32.453+0000 7f1d34e41700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f1d3010cfb0 con 0x7f1d3006fbd0 2023-11-17T05:21:26.196 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:32.453+0000 7f1d2e59c700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f1d3006fbd0 0x7f1d3010cac0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:21:26.196 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:32.453+0000 7f1d2e59c700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f1d3006fbd0 0x7f1d3010cac0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:54412/0 (socket says 172.21.15.40:54412) 2023-11-17T05:21:26.196 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:32.453+0000 7f1d2e59c700 1 -- 172.21.15.40:0/1268000519 learned_addr learned my addr 172.21.15.40:0/1268000519 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-11-17T05:21:26.196 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:32.453+0000 7f1d2e59c700 1 -- 172.21.15.40:0/1268000519 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f1d2000b200 con 0x7f1d3006fbd0 2023-11-17T05:21:26.196 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:32.453+0000 7f1d2e59c700 1 --2- 172.21.15.40:0/1268000519 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f1d3006fbd0 0x7f1d3010cac0 secure :-1 s=READY pgs=23 cs=0 l=1 rev1=1 rx=0x7f1d20011820 tx=0x7f1d20006ee0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:21:26.197 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:32.453+0000 7f1d1effd700 1 -- 172.21.15.40:0/1268000519 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f1d2001b410 con 0x7f1d3006fbd0 2023-11-17T05:21:26.197 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:32.453+0000 7f1d1effd700 1 -- 172.21.15.40:0/1268000519 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(25 keys) v1 ==== 1010+0+0 (secure 0 0 0) 0x7f1d2001b9e0 con 0x7f1d3006fbd0 2023-11-17T05:21:26.197 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:32.453+0000 7f1d34e41700 1 -- 172.21.15.40:0/1268000519 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f1d3010d1b0 con 0x7f1d3006fbd0 2023-11-17T05:21:26.197 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:32.454+0000 7f1d34e41700 1 -- 172.21.15.40:0/1268000519 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f1d3010d640 con 0x7f1d3006fbd0 2023-11-17T05:21:26.197 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:32.454+0000 7f1d1effd700 1 -- 172.21.15.40:0/1268000519 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f1d2001a8b0 con 0x7f1d3006fbd0 2023-11-17T05:21:26.198 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:32.455+0000 7f1d1effd700 1 -- 172.21.15.40:0/1268000519 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 5) v1 ==== 36120+0+0 (secure 0 0 0) 0x7f1d2000f070 con 0x7f1d3006fbd0 2023-11-17T05:21:26.198 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:32.455+0000 7f1d34e41700 1 -- 172.21.15.40:0/1268000519 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f1d0c004fa0 con 0x7f1d3006fbd0 2023-11-17T05:21:26.198 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:32.455+0000 7f1d1effd700 1 --2- 172.21.15.40:0/1268000519 >> [v2:172.21.15.40:6800/74978488,v1:172.21.15.40:6801/74978488] conn(0x7f1d1402fdb0 0x7f1d14032270 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:21:26.199 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:32.455+0000 7f1d1effd700 1 -- 172.21.15.40:0/1268000519 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(1..1 src has 1..1) v4 ==== 754+0+0 (secure 0 0 0) 0x7f1d20041230 con 0x7f1d3006fbd0 2023-11-17T05:21:26.199 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:32.455+0000 7f1d2dd9b700 1 -- 172.21.15.40:0/1268000519 >> [v2:172.21.15.40:6800/74978488,v1:172.21.15.40:6801/74978488] conn(0x7f1d1402fdb0 msgr2=0x7f1d14032270 unknown :-1 s=STATE_CONNECTING_RE l=1).process reconnect failed to v2:172.21.15.40:6800/74978488 2023-11-17T05:21:26.199 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:32.455+0000 7f1d2dd9b700 1 --2- 172.21.15.40:0/1268000519 >> [v2:172.21.15.40:6800/74978488,v1:172.21.15.40:6801/74978488] conn(0x7f1d1402fdb0 0x7f1d14032270 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._fault waiting 0.200000 2023-11-17T05:21:26.199 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:32.460+0000 7f1d1effd700 1 -- 172.21.15.40:0/1268000519 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+152212 (secure 0 0 0) 0x7f1d20022d20 con 0x7f1d3006fbd0 2023-11-17T05:21:26.199 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:32.656+0000 7f1d2dd9b700 1 -- 172.21.15.40:0/1268000519 >> [v2:172.21.15.40:6800/74978488,v1:172.21.15.40:6801/74978488] conn(0x7f1d1402fdb0 msgr2=0x7f1d14032270 unknown :-1 s=STATE_CONNECTING_RE l=1).process reconnect failed to v2:172.21.15.40:6800/74978488 2023-11-17T05:21:26.199 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:32.656+0000 7f1d2dd9b700 1 --2- 172.21.15.40:0/1268000519 >> [v2:172.21.15.40:6800/74978488,v1:172.21.15.40:6801/74978488] conn(0x7f1d1402fdb0 0x7f1d14032270 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._fault waiting 0.400000 2023-11-17T05:21:26.202 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:32.676+0000 7f1d34e41700 1 -- 172.21.15.40:0/1268000519 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "mgr stat"} v 0) v1 -- 0x7f1d0c005e80 con 0x7f1d3006fbd0 2023-11-17T05:21:26.202 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:32.677+0000 7f1d1effd700 1 -- 172.21.15.40:0/1268000519 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mon_command_ack([{"prefix": "mgr stat"}]=0 v5) v1 ==== 56+0+103 (secure 0 0 0) 0x7f1d20016070 con 0x7f1d3006fbd0 2023-11-17T05:21:26.202 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: { 2023-11-17T05:21:26.203 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: "epoch": 5, 2023-11-17T05:21:26.203 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: "available": true, 2023-11-17T05:21:26.203 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: "active_name": "smithi040.ogcprl", 2023-11-17T05:21:26.203 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: "num_standby": 0 2023-11-17T05:21:26.203 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: } 2023-11-17T05:21:26.203 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:32.679+0000 7f1d34e41700 1 -- 172.21.15.40:0/1268000519 >> [v2:172.21.15.40:6800/74978488,v1:172.21.15.40:6801/74978488] conn(0x7f1d1402fdb0 msgr2=0x7f1d14032270 unknown :-1 s=STATE_CONNECTING l=1).mark_down 2023-11-17T05:21:26.204 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:32.679+0000 7f1d34e41700 1 --2- 172.21.15.40:0/1268000519 >> [v2:172.21.15.40:6800/74978488,v1:172.21.15.40:6801/74978488] conn(0x7f1d1402fdb0 0x7f1d14032270 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).stop 2023-11-17T05:21:26.204 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:32.679+0000 7f1d34e41700 1 -- 172.21.15.40:0/1268000519 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f1d3006fbd0 msgr2=0x7f1d3010cac0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:21:26.204 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:32.679+0000 7f1d34e41700 1 --2- 172.21.15.40:0/1268000519 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f1d3006fbd0 0x7f1d3010cac0 secure :-1 s=READY pgs=23 cs=0 l=1 rev1=1 rx=0x7f1d20011820 tx=0x7f1d20006ee0).stop 2023-11-17T05:21:26.204 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:32.679+0000 7f1d34e41700 1 -- 172.21.15.40:0/1268000519 shutdown_connections 2023-11-17T05:21:26.204 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:32.679+0000 7f1d34e41700 1 --2- 172.21.15.40:0/1268000519 >> [v2:172.21.15.40:6800/74978488,v1:172.21.15.40:6801/74978488] conn(0x7f1d1402fdb0 0x7f1d14032270 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).stop 2023-11-17T05:21:26.204 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:32.679+0000 7f1d34e41700 1 --2- 172.21.15.40:0/1268000519 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f1d3006fbd0 0x7f1d3010cac0 unknown :-1 s=CLOSED pgs=23 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:21:26.205 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:32.679+0000 7f1d34e41700 1 -- 172.21.15.40:0/1268000519 >> 172.21.15.40:0/1268000519 conn(0x7f1d300f6270 msgr2=0x7f1d300f7d80 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:21:26.205 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:32.679+0000 7f1d34e41700 1 -- 172.21.15.40:0/1268000519 shutdown_connections 2023-11-17T05:21:26.205 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:32.679+0000 7f1d34e41700 1 -- 172.21.15.40:0/1268000519 wait complete. 2023-11-17T05:21:26.206 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:32 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:20:31.472584+0000 mon.smithi040 (mon.0) 27 : audit [INF] from='client.? 172.21.15.40:0/1801812697' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "cephadm"}]': finished 2023-11-17T05:21:26.206 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:32 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:20:31.472664+0000 mon.smithi040 (mon.0) 28 : cluster [DBG] mgrmap e5: smithi040.ogcprl(active, since 5s) 2023-11-17T05:21:26.206 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:33 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:20:32.677854+0000 mon.smithi040 (mon.0) 29 : audit [DBG] from='client.? 172.21.15.40:0/1268000519' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2023-11-17T05:21:26.214 INFO:teuthology.orchestra.run.smithi040.stderr:Waiting for the mgr to restart... 2023-11-17T05:21:26.214 INFO:teuthology.orchestra.run.smithi040.stderr:Waiting for mgr epoch 5... 2023-11-17T05:21:26.214 INFO:teuthology.orchestra.run.smithi040.stderr:Running command: /bin/podman run --rm --ipc=host --no-hosts --net=host --entrypoint /usr/bin/ceph --init -e CONTAINER_IMAGE=docker.io/ceph/ceph:v16.2.4 -e NODE_NAME=smithi040 -e CEPH_USE_RANDOM_NONCE=1 -v /var/log/ceph/ed6470e2-8508-11ee-8dbe-212e2dc638e7:/var/log/ceph:z -v /tmp/ceph-tmpzx79_7_h:/etc/ceph/ceph.client.admin.keyring:z -v /tmp/ceph-tmpru9tjirw:/etc/ceph/ceph.conf:z docker.io/ceph/ceph:v16.2.4 tell mgr mgr_status 2023-11-17T05:21:26.214 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:35.586+0000 7f0b1fd9b700 1 Processor -- start 2023-11-17T05:21:26.214 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:35.587+0000 7f0b1fd9b700 1 -- start start 2023-11-17T05:21:26.215 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:35.587+0000 7f0b1fd9b700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f0b180faae0 0x7f0b180faf00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 rx=0 tx=0).connect 2023-11-17T05:21:26.215 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:35.587+0000 7f0b1fd9b700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f0b180fb480 con 0x7f0b180faae0 2023-11-17T05:21:26.215 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:35.587+0000 7f0b1db37700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f0b180faae0 0x7f0b180faf00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:21:26.215 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:35.587+0000 7f0b1db37700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f0b180faae0 0x7f0b180faf00 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:51014/0 (socket says 172.21.15.40:51014) 2023-11-17T05:21:26.215 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:35.587+0000 7f0b1db37700 1 -- 172.21.15.40:0/432994069 learned_addr learned my addr 172.21.15.40:0/432994069 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-11-17T05:21:26.215 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:35.588+0000 7f0b1db37700 1 -- 172.21.15.40:0/432994069 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f0b180fb5c0 con 0x7f0b180faae0 2023-11-17T05:21:26.216 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:35.588+0000 7f0b1db37700 1 --2- 172.21.15.40:0/432994069 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f0b180faae0 0x7f0b180faf00 secure :-1 s=READY pgs=24 cs=0 l=1 rev1=1 rx=0x7f0b1400b5f0 tx=0x7f0b14006ee0).ready entity=mon.0 client_cookie=85c4a829cce82d40 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:21:26.216 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:35.588+0000 7f0b1cb35700 1 -- 172.21.15.40:0/432994069 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f0b1400d600 con 0x7f0b180faae0 2023-11-17T05:21:26.216 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:35.588+0000 7f0b1cb35700 1 -- 172.21.15.40:0/432994069 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(25 keys) v1 ==== 1010+0+0 (secure 0 0 0) 0x7f0b1400dbd0 con 0x7f0b180faae0 2023-11-17T05:21:26.216 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:35.588+0000 7f0b1cb35700 1 -- 172.21.15.40:0/432994069 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f0b14011c00 con 0x7f0b180faae0 2023-11-17T05:21:26.216 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:35.589+0000 7f0b1fd9b700 1 -- 172.21.15.40:0/432994069 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f0b180faae0 msgr2=0x7f0b180faf00 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:21:26.216 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:35.589+0000 7f0b1fd9b700 1 --2- 172.21.15.40:0/432994069 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f0b180faae0 0x7f0b180faf00 secure :-1 s=READY pgs=24 cs=0 l=1 rev1=1 rx=0x7f0b1400b5f0 tx=0x7f0b14006ee0).stop 2023-11-17T05:21:26.217 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:35.589+0000 7f0b1fd9b700 1 -- 172.21.15.40:0/432994069 shutdown_connections 2023-11-17T05:21:26.217 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:35.589+0000 7f0b1fd9b700 1 --2- 172.21.15.40:0/432994069 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f0b180faae0 0x7f0b180faf00 unknown :-1 s=CLOSED pgs=24 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:21:26.217 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:35.589+0000 7f0b1fd9b700 1 -- 172.21.15.40:0/432994069 >> 172.21.15.40:0/432994069 conn(0x7f0b180f6280 msgr2=0x7f0b180f86a0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:21:26.217 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:35.589+0000 7f0b1fd9b700 1 -- 172.21.15.40:0/432994069 shutdown_connections 2023-11-17T05:21:26.217 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:35.589+0000 7f0b1fd9b700 1 -- 172.21.15.40:0/432994069 wait complete. 2023-11-17T05:21:26.217 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:35.589+0000 7f0b1fd9b700 1 Processor -- start 2023-11-17T05:21:26.218 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:35.589+0000 7f0b1fd9b700 1 -- start start 2023-11-17T05:21:26.218 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:35.590+0000 7f0b1fd9b700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f0b180faae0 0x7f0b18100df0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:21:26.218 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:35.590+0000 7f0b1fd9b700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f0b180ff270 con 0x7f0b180faae0 2023-11-17T05:21:26.218 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:35.590+0000 7f0b1db37700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f0b180faae0 0x7f0b18100df0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:21:26.218 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:35.590+0000 7f0b1db37700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f0b180faae0 0x7f0b18100df0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:51028/0 (socket says 172.21.15.40:51028) 2023-11-17T05:21:26.218 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:35.590+0000 7f0b1db37700 1 -- 172.21.15.40:0/986189408 learned_addr learned my addr 172.21.15.40:0/986189408 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-11-17T05:21:26.219 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:35.590+0000 7f0b1db37700 1 -- 172.21.15.40:0/986189408 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f0b1400b200 con 0x7f0b180faae0 2023-11-17T05:21:26.219 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:35.590+0000 7f0b1db37700 1 --2- 172.21.15.40:0/986189408 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f0b180faae0 0x7f0b18100df0 secure :-1 s=READY pgs=25 cs=0 l=1 rev1=1 rx=0x7f0b1400b1d0 tx=0x7f0b14006ee0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:21:26.219 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:35.590+0000 7f0b067fc700 1 -- 172.21.15.40:0/986189408 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f0b14011a20 con 0x7f0b180faae0 2023-11-17T05:21:26.219 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:35.590+0000 7f0b067fc700 1 -- 172.21.15.40:0/986189408 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(25 keys) v1 ==== 1010+0+0 (secure 0 0 0) 0x7f0b1401b880 con 0x7f0b180faae0 2023-11-17T05:21:26.219 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:35.590+0000 7f0b1fd9b700 1 -- 172.21.15.40:0/986189408 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f0b180ff410 con 0x7f0b180faae0 2023-11-17T05:21:26.219 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:35.590+0000 7f0b067fc700 1 -- 172.21.15.40:0/986189408 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f0b1401a700 con 0x7f0b180faae0 2023-11-17T05:21:26.220 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:35.590+0000 7f0b1fd9b700 1 -- 172.21.15.40:0/986189408 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f0b180ff820 con 0x7f0b180faae0 2023-11-17T05:21:26.221 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:35.591+0000 7f0b067fc700 1 -- 172.21.15.40:0/986189408 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 5) v1 ==== 36120+0+0 (secure 0 0 0) 0x7f0b1400f070 con 0x7f0b180faae0 2023-11-17T05:21:26.221 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:35.591+0000 7f0b067fc700 1 --2- 172.21.15.40:0/986189408 >> [v2:172.21.15.40:6800/74978488,v1:172.21.15.40:6801/74978488] conn(0x7f0b0002fd50 0x7f0b00032210 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:21:26.221 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:35.591+0000 7f0b1fd9b700 1 -- 172.21.15.40:0/986189408 --> [v2:172.21.15.40:6800/74978488,v1:172.21.15.40:6801/74978488] -- command(tid 0: {"prefix": "get_command_descriptions"}) v1 -- 0x7f0b18059100 con 0x7f0b0002fd50 2023-11-17T05:21:26.221 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:35.591+0000 7f0b067fc700 1 -- 172.21.15.40:0/986189408 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(1..1 src has 1..1) v4 ==== 754+0+0 (secure 0 0 0) 0x7f0b14042000 con 0x7f0b180faae0 2023-11-17T05:21:26.222 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:35.591+0000 7f0b1d336700 1 -- 172.21.15.40:0/986189408 >> [v2:172.21.15.40:6800/74978488,v1:172.21.15.40:6801/74978488] conn(0x7f0b0002fd50 msgr2=0x7f0b00032210 unknown :-1 s=STATE_CONNECTING_RE l=1).process reconnect failed to v2:172.21.15.40:6800/74978488 2023-11-17T05:21:26.222 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:35.592+0000 7f0b1d336700 1 --2- 172.21.15.40:0/986189408 >> [v2:172.21.15.40:6800/74978488,v1:172.21.15.40:6801/74978488] conn(0x7f0b0002fd50 0x7f0b00032210 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._fault waiting 0.200000 2023-11-17T05:21:26.222 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:35.793+0000 7f0b1d336700 1 -- 172.21.15.40:0/986189408 >> [v2:172.21.15.40:6800/74978488,v1:172.21.15.40:6801/74978488] conn(0x7f0b0002fd50 msgr2=0x7f0b00032210 unknown :-1 s=STATE_CONNECTING_RE l=1).process reconnect failed to v2:172.21.15.40:6800/74978488 2023-11-17T05:21:26.222 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:35.793+0000 7f0b1d336700 1 --2- 172.21.15.40:0/986189408 >> [v2:172.21.15.40:6800/74978488,v1:172.21.15.40:6801/74978488] conn(0x7f0b0002fd50 0x7f0b00032210 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._fault waiting 0.400000 2023-11-17T05:21:26.222 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:36.194+0000 7f0b1d336700 1 -- 172.21.15.40:0/986189408 >> [v2:172.21.15.40:6800/74978488,v1:172.21.15.40:6801/74978488] conn(0x7f0b0002fd50 msgr2=0x7f0b00032210 unknown :-1 s=STATE_CONNECTING_RE l=1).process reconnect failed to v2:172.21.15.40:6800/74978488 2023-11-17T05:21:26.222 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:36.194+0000 7f0b1d336700 1 --2- 172.21.15.40:0/986189408 >> [v2:172.21.15.40:6800/74978488,v1:172.21.15.40:6801/74978488] conn(0x7f0b0002fd50 0x7f0b00032210 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._fault waiting 0.800000 2023-11-17T05:21:26.223 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:36.995+0000 7f0b1d336700 1 -- 172.21.15.40:0/986189408 >> [v2:172.21.15.40:6800/74978488,v1:172.21.15.40:6801/74978488] conn(0x7f0b0002fd50 msgr2=0x7f0b00032210 unknown :-1 s=STATE_CONNECTING_RE l=1).process reconnect failed to v2:172.21.15.40:6800/74978488 2023-11-17T05:21:26.223 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:36.995+0000 7f0b1d336700 1 --2- 172.21.15.40:0/986189408 >> [v2:172.21.15.40:6800/74978488,v1:172.21.15.40:6801/74978488] conn(0x7f0b0002fd50 0x7f0b00032210 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._fault waiting 1.600000 2023-11-17T05:21:26.223 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:37.131+0000 7f0b067fc700 1 -- 172.21.15.40:0/986189408 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mgrmap(e 6) v1 ==== 35970+0+0 (secure 0 0 0) 0x7f0b140257e0 con 0x7f0b180faae0 2023-11-17T05:21:26.230 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:37.131+0000 7f0b067fc700 1 -- 172.21.15.40:0/986189408 >> [v2:172.21.15.40:6800/74978488,v1:172.21.15.40:6801/74978488] conn(0x7f0b0002fd50 msgr2=0x7f0b00032210 unknown :-1 s=STATE_CONNECTING l=1).mark_down 2023-11-17T05:21:26.230 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:37.131+0000 7f0b067fc700 1 --2- 172.21.15.40:0/986189408 >> [v2:172.21.15.40:6800/74978488,v1:172.21.15.40:6801/74978488] conn(0x7f0b0002fd50 0x7f0b00032210 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).stop 2023-11-17T05:21:26.231 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:37 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:20:37.079540+0000 mon.smithi040 (mon.0) 30 : cluster [INF] Active manager daemon smithi040.ogcprl restarted 2023-11-17T05:21:26.231 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:37 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:20:37.079978+0000 mon.smithi040 (mon.0) 31 : cluster [INF] Activating manager daemon smithi040.ogcprl 2023-11-17T05:21:26.231 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:37 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:20:37.082284+0000 mon.smithi040 (mon.0) 32 : cluster [DBG] osdmap e2: 0 total, 0 up, 0 in 2023-11-17T05:21:26.233 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:38.139+0000 7f0b067fc700 1 -- 172.21.15.40:0/986189408 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mgrmap(e 7) v1 ==== 36040+0+0 (secure 0 0 0) 0x7f0b1401b9f0 con 0x7f0b180faae0 2023-11-17T05:21:26.233 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:38.140+0000 7f0b067fc700 1 --2- 172.21.15.40:0/986189408 >> [v2:172.21.15.40:6800/929662169,v1:172.21.15.40:6801/929662169] conn(0x7f0b000336c0 0x7f0b00035ab0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:21:26.233 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:38.140+0000 7f0b067fc700 1 -- 172.21.15.40:0/986189408 --> [v2:172.21.15.40:6800/929662169,v1:172.21.15.40:6801/929662169] -- command(tid 0: {"prefix": "get_command_descriptions"}) v1 -- 0x7f0b18059100 con 0x7f0b000336c0 2023-11-17T05:21:26.233 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:38.140+0000 7f0b1d336700 1 --2- 172.21.15.40:0/986189408 >> [v2:172.21.15.40:6800/929662169,v1:172.21.15.40:6801/929662169] conn(0x7f0b000336c0 0x7f0b00035ab0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:21:26.233 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:38.141+0000 7f0b1d336700 1 --2- 172.21.15.40:0/986189408 >> [v2:172.21.15.40:6800/929662169,v1:172.21.15.40:6801/929662169] conn(0x7f0b000336c0 0x7f0b00035ab0 secure :-1 s=READY pgs=1 cs=0 l=1 rev1=1 rx=0x7f0b080036d0 tx=0x7f0b08006ca0).ready entity=mgr.14116 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:21:26.233 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:38.142+0000 7f0b067fc700 1 -- 172.21.15.40:0/986189408 <== mgr.14116 v2:172.21.15.40:6800/929662169 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+6438 (secure 0 0 0) 0x7f0b18059100 con 0x7f0b000336c0 2023-11-17T05:21:26.233 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:38.149+0000 7f0b1fd9b700 1 -- 172.21.15.40:0/986189408 --> [v2:172.21.15.40:6800/929662169,v1:172.21.15.40:6801/929662169] -- command(tid 1: {"prefix": "mgr_status"}) v1 -- 0x7f0b18045ad0 con 0x7f0b000336c0 2023-11-17T05:21:26.233 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:38.149+0000 7f0b067fc700 1 -- 172.21.15.40:0/986189408 <== mgr.14116 v2:172.21.15.40:6800/929662169 2 ==== command_reply(tid 1: 0 ) v1 ==== 8+0+51 (secure 0 0 0) 0x7f0b18045ad0 con 0x7f0b000336c0 2023-11-17T05:21:26.233 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: { 2023-11-17T05:21:26.233 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: "mgrmap_epoch": 7, 2023-11-17T05:21:26.234 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: "initialized": true 2023-11-17T05:21:26.234 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: } 2023-11-17T05:21:26.234 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:38.149+0000 7f0b1fd9b700 1 -- 172.21.15.40:0/986189408 >> [v2:172.21.15.40:6800/929662169,v1:172.21.15.40:6801/929662169] conn(0x7f0b000336c0 msgr2=0x7f0b00035ab0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:21:26.234 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:38.149+0000 7f0b1fd9b700 1 --2- 172.21.15.40:0/986189408 >> [v2:172.21.15.40:6800/929662169,v1:172.21.15.40:6801/929662169] conn(0x7f0b000336c0 0x7f0b00035ab0 secure :-1 s=READY pgs=1 cs=0 l=1 rev1=1 rx=0x7f0b080036d0 tx=0x7f0b08006ca0).stop 2023-11-17T05:21:26.234 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:38.149+0000 7f0b1fd9b700 1 -- 172.21.15.40:0/986189408 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f0b180faae0 msgr2=0x7f0b18100df0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:21:26.234 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:38.149+0000 7f0b1fd9b700 1 --2- 172.21.15.40:0/986189408 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f0b180faae0 0x7f0b18100df0 secure :-1 s=READY pgs=25 cs=0 l=1 rev1=1 rx=0x7f0b1400b1d0 tx=0x7f0b14006ee0).stop 2023-11-17T05:21:26.234 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:38.149+0000 7f0b1fd9b700 1 -- 172.21.15.40:0/986189408 shutdown_connections 2023-11-17T05:21:26.234 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:38.149+0000 7f0b1fd9b700 1 --2- 172.21.15.40:0/986189408 >> [v2:172.21.15.40:6800/929662169,v1:172.21.15.40:6801/929662169] conn(0x7f0b000336c0 0x7f0b00035ab0 unknown :-1 s=CLOSED pgs=1 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:21:26.234 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:38.149+0000 7f0b1fd9b700 1 --2- 172.21.15.40:0/986189408 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f0b180faae0 0x7f0b18100df0 unknown :-1 s=CLOSED pgs=25 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:21:26.234 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:38.149+0000 7f0b1fd9b700 1 --2- 172.21.15.40:0/986189408 >> [v2:172.21.15.40:6800/74978488,v1:172.21.15.40:6801/74978488] conn(0x7f0b0002fd50 0x7f0b00032210 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).stop 2023-11-17T05:21:26.234 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:38.149+0000 7f0b1fd9b700 1 -- 172.21.15.40:0/986189408 >> 172.21.15.40:0/986189408 conn(0x7f0b180f6280 msgr2=0x7f0b1806f850 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:21:26.234 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:38.150+0000 7f0b1fd9b700 1 -- 172.21.15.40:0/986189408 shutdown_connections 2023-11-17T05:21:26.234 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:38.150+0000 7f0b1fd9b700 1 -- 172.21.15.40:0/986189408 wait complete. 2023-11-17T05:21:26.235 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:38 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:20:37.132800+0000 mon.smithi040 (mon.0) 33 : cluster [DBG] mgrmap e6: smithi040.ogcprl(active, starting, since 0.0528975s) 2023-11-17T05:21:26.236 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:38 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:20:37.136262+0000 mon.smithi040 (mon.0) 34 : audit [DBG] from='mgr.14116 172.21.15.40:0/860568587' entity='mgr.smithi040.ogcprl' cmd=[{"prefix": "mon metadata", "id": "smithi040"}]: dispatch 2023-11-17T05:21:26.236 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:38 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:20:37.136488+0000 mon.smithi040 (mon.0) 35 : audit [DBG] from='mgr.14116 172.21.15.40:0/860568587' entity='mgr.smithi040.ogcprl' cmd=[{"prefix": "mgr metadata", "who": "smithi040.ogcprl", "id": "smithi040.ogcprl"}]: dispatch 2023-11-17T05:21:26.236 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:38 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:20:37.136712+0000 mon.smithi040 (mon.0) 36 : audit [DBG] from='mgr.14116 172.21.15.40:0/860568587' entity='mgr.smithi040.ogcprl' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-11-17T05:21:26.236 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:38 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:20:37.136877+0000 mon.smithi040 (mon.0) 37 : audit [DBG] from='mgr.14116 172.21.15.40:0/860568587' entity='mgr.smithi040.ogcprl' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-11-17T05:21:26.236 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:38 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:20:37.137022+0000 mon.smithi040 (mon.0) 38 : audit [DBG] from='mgr.14116 172.21.15.40:0/860568587' entity='mgr.smithi040.ogcprl' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-11-17T05:21:26.236 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:38 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:20:37.151244+0000 mon.smithi040 (mon.0) 39 : cluster [INF] Manager daemon smithi040.ogcprl is now available 2023-11-17T05:21:26.236 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:38 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:20:37.167783+0000 mon.smithi040 (mon.0) 40 : audit [INF] from='mgr.14116 172.21.15.40:0/860568587' entity='mgr.smithi040.ogcprl' 2023-11-17T05:21:26.236 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:38 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:20:37.170856+0000 mon.smithi040 (mon.0) 41 : audit [INF] from='mgr.14116 172.21.15.40:0/860568587' entity='mgr.smithi040.ogcprl' 2023-11-17T05:21:26.236 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:38 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:20:37.173609+0000 mon.smithi040 (mon.0) 42 : audit [INF] from='mgr.14116 172.21.15.40:0/860568587' entity='mgr.smithi040.ogcprl' 2023-11-17T05:21:26.236 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:38 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:20:37.276801+0000 mon.smithi040 (mon.0) 43 : audit [INF] from='mgr.14116 172.21.15.40:0/860568587' entity='mgr.smithi040.ogcprl' 2023-11-17T05:21:26.236 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:38 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:20:37.277586+0000 mon.smithi040 (mon.0) 44 : audit [DBG] from='mgr.14116 172.21.15.40:0/860568587' entity='mgr.smithi040.ogcprl' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-11-17T05:21:26.236 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:38 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:20:37.278876+0000 mon.smithi040 (mon.0) 45 : audit [DBG] from='mgr.14116 172.21.15.40:0/860568587' entity='mgr.smithi040.ogcprl' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-11-17T05:21:26.237 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:38 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:20:37.280474+0000 mon.smithi040 (mon.0) 46 : audit [DBG] from='mgr.14116 172.21.15.40:0/860568587' entity='mgr.smithi040.ogcprl' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-11-17T05:21:26.237 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:38 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:20:37.293021+0000 mon.smithi040 (mon.0) 47 : audit [INF] from='mgr.14116 172.21.15.40:0/860568587' entity='mgr.smithi040.ogcprl' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi040.ogcprl/mirror_snapshot_schedule"}]: dispatch 2023-11-17T05:21:26.237 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:38 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:20:37.294461+0000 mon.smithi040 (mon.0) 48 : audit [INF] from='mgr.14116 172.21.15.40:0/860568587' entity='mgr.smithi040.ogcprl' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi040.ogcprl/trash_purge_schedule"}]: dispatch 2023-11-17T05:21:26.237 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:39 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:20:38.142276+0000 mgr.smithi040.ogcprl (mgr.14116) 1 : audit [DBG] from='client.14120 -' entity='client.admin' cmd=[{"prefix": "get_command_descriptions"}]: dispatch 2023-11-17T05:21:26.237 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:39 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:20:38.142848+0000 mon.smithi040 (mon.0) 49 : cluster [DBG] mgrmap e7: smithi040.ogcprl(active, since 1.06294s) 2023-11-17T05:21:26.237 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:39 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:20:38.149836+0000 mgr.smithi040.ogcprl (mgr.14116) 2 : audit [DBG] from='client.14120 -' entity='client.admin' cmd=[{"prefix": "mgr_status"}]: dispatch 2023-11-17T05:21:26.237 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:39 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:20:39.131306+0000 mon.smithi040 (mon.0) 50 : audit [INF] from='mgr.14116 172.21.15.40:0/860568587' entity='mgr.smithi040.ogcprl' 2023-11-17T05:21:26.237 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:39 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:20:39.135285+0000 mon.smithi040 (mon.0) 51 : audit [DBG] from='mgr.14116 172.21.15.40:0/860568587' entity='mgr.smithi040.ogcprl' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-11-17T05:21:26.237 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:40 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:20:39.126917+0000 mgr.smithi040.ogcprl (mgr.14116) 3 : audit [DBG] from='client.14126 -' entity='client.admin' cmd=[{"prefix": "orch set backend", "module_name": "cephadm", "target": ["mon-mgr", ""]}]: dispatch 2023-11-17T05:21:26.237 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:40 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:20:39.142602+0000 mon.smithi040 (mon.0) 52 : cluster [DBG] mgrmap e8: smithi040.ogcprl(active, since 2s) 2023-11-17T05:21:26.237 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:41 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:20:40.069747+0000 mgr.smithi040.ogcprl (mgr.14116) 4 : audit [DBG] from='client.14128 -' entity='client.admin' cmd=[{"prefix": "cephadm set-user", "user": "root", "target": ["mon-mgr", ""]}]: dispatch 2023-11-17T05:21:26.237 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:42 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cephadm 2023-11-17T05:20:40.965084+0000 mgr.smithi040.ogcprl (mgr.14116) 6 : cephadm [INF] Generating ssh key... 2023-11-17T05:21:26.237 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:42 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:20:41.823217+0000 mon.smithi040 (mon.0) 53 : audit [INF] from='mgr.14116 172.21.15.40:0/860568587' entity='mgr.smithi040.ogcprl' 2023-11-17T05:21:26.238 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:42 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:20:41.825057+0000 mon.smithi040 (mon.0) 54 : audit [INF] from='mgr.14116 172.21.15.40:0/860568587' entity='mgr.smithi040.ogcprl' 2023-11-17T05:21:26.238 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:46 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:20:45.651159+0000 mgr.smithi040.ogcprl (mgr.14116) 7 : audit [DBG] from='client.14132 -' entity='client.admin' cmd=[{"prefix": "cephadm get-pub-key", "target": ["mon-mgr", ""]}]: dispatch 2023-11-17T05:21:26.238 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:48 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:20:48.075153+0000 mgr.smithi040.ogcprl (mgr.14116) 8 : audit [DBG] from='client.14134 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi040", "addr": "172.21.15.40", "target": ["mon-mgr", ""]}]: dispatch 2023-11-17T05:21:26.238 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:49 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:20:48.737919+0000 mon.smithi040 (mon.0) 55 : audit [INF] from='mgr.14116 172.21.15.40:0/860568587' entity='mgr.smithi040.ogcprl' 2023-11-17T05:21:26.238 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:49 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cephadm 2023-11-17T05:20:48.738259+0000 mgr.smithi040.ogcprl (mgr.14116) 9 : cephadm [INF] Added host smithi040 2023-11-17T05:21:26.238 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:49 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:20:48.738531+0000 mon.smithi040 (mon.0) 56 : audit [DBG] from='mgr.14116 172.21.15.40:0/860568587' entity='mgr.smithi040.ogcprl' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-11-17T05:21:26.238 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:49 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:20:49.637491+0000 mon.smithi040 (mon.0) 57 : audit [INF] from='mgr.14116 172.21.15.40:0/860568587' entity='mgr.smithi040.ogcprl' 2023-11-17T05:21:26.238 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:50 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:20:49.633357+0000 mgr.smithi040.ogcprl (mgr.14116) 10 : audit [DBG] from='client.14136 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "target": ["mon-mgr", ""]}]: dispatch 2023-11-17T05:21:26.238 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:50 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cephadm 2023-11-17T05:20:49.634202+0000 mgr.smithi040.ogcprl (mgr.14116) 11 : cephadm [INF] Saving service mon spec with placement count:5 2023-11-17T05:21:26.238 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:50 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:20:50.606846+0000 mon.smithi040 (mon.0) 58 : audit [INF] from='mgr.14116 172.21.15.40:0/860568587' entity='mgr.smithi040.ogcprl' 2023-11-17T05:21:26.238 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:51 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:20:50.603803+0000 mgr.smithi040.ogcprl (mgr.14116) 12 : audit [DBG] from='client.14138 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "target": ["mon-mgr", ""]}]: dispatch 2023-11-17T05:21:26.238 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:51 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cephadm 2023-11-17T05:20:50.604383+0000 mgr.smithi040.ogcprl (mgr.14116) 13 : cephadm [INF] Saving service mgr spec with placement count:2 2023-11-17T05:21:26.238 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:51 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:20:51.567564+0000 mon.smithi040 (mon.0) 59 : audit [INF] from='mgr.14116 172.21.15.40:0/860568587' entity='mgr.smithi040.ogcprl' 2023-11-17T05:21:26.239 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:52 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:20:51.564459+0000 mgr.smithi040.ogcprl (mgr.14116) 14 : audit [DBG] from='client.14140 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "crash", "target": ["mon-mgr", ""]}]: dispatch 2023-11-17T05:21:26.239 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:52 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cephadm 2023-11-17T05:20:51.565166+0000 mgr.smithi040.ogcprl (mgr.14116) 15 : cephadm [INF] Saving service crash spec with placement * 2023-11-17T05:21:26.239 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:52 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:20:52.537508+0000 mon.smithi040 (mon.0) 60 : audit [INF] from='client.? 172.21.15.40:0/4203187453' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2023-11-17T05:21:26.239 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:53 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:20:52.744232+0000 mon.smithi040 (mon.0) 61 : audit [INF] from='client.? 172.21.15.40:0/4203187453' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2023-11-17T05:21:26.239 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:53 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:20:52.744287+0000 mon.smithi040 (mon.0) 62 : cluster [DBG] mgrmap e9: smithi040.ogcprl(active, since 15s) 2023-11-17T05:21:26.239 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:58 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:20:58.360581+0000 mon.smithi040 (mon.0) 63 : cluster [INF] Active manager daemon smithi040.ogcprl restarted 2023-11-17T05:21:26.253 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:58 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:20:58.361017+0000 mon.smithi040 (mon.0) 64 : cluster [INF] Activating manager daemon smithi040.ogcprl 2023-11-17T05:21:26.253 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:58 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:20:58.365496+0000 mon.smithi040 (mon.0) 65 : cluster [DBG] osdmap e3: 0 total, 0 up, 0 in 2023-11-17T05:21:26.253 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:59 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:20:58.416394+0000 mon.smithi040 (mon.0) 66 : cluster [DBG] mgrmap e10: smithi040.ogcprl(active, starting, since 0.0554758s) 2023-11-17T05:21:26.253 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:59 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:20:58.419718+0000 mon.smithi040 (mon.0) 67 : audit [DBG] from='mgr.14144 172.21.15.40:0/516882453' entity='mgr.smithi040.ogcprl' cmd=[{"prefix": "mon metadata", "id": "smithi040"}]: dispatch 2023-11-17T05:21:26.254 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:59 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:20:58.419955+0000 mon.smithi040 (mon.0) 68 : audit [DBG] from='mgr.14144 172.21.15.40:0/516882453' entity='mgr.smithi040.ogcprl' cmd=[{"prefix": "mgr metadata", "who": "smithi040.ogcprl", "id": "smithi040.ogcprl"}]: dispatch 2023-11-17T05:21:26.254 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:59 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:20:58.420165+0000 mon.smithi040 (mon.0) 69 : audit [DBG] from='mgr.14144 172.21.15.40:0/516882453' entity='mgr.smithi040.ogcprl' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-11-17T05:21:26.254 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:59 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:20:58.420340+0000 mon.smithi040 (mon.0) 70 : audit [DBG] from='mgr.14144 172.21.15.40:0/516882453' entity='mgr.smithi040.ogcprl' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-11-17T05:21:26.254 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:59 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:20:58.420500+0000 mon.smithi040 (mon.0) 71 : audit [DBG] from='mgr.14144 172.21.15.40:0/516882453' entity='mgr.smithi040.ogcprl' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-11-17T05:21:26.254 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:59 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:20:58.434350+0000 mon.smithi040 (mon.0) 72 : cluster [INF] Manager daemon smithi040.ogcprl is now available 2023-11-17T05:21:26.254 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:59 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:20:58.548378+0000 mon.smithi040 (mon.0) 73 : audit [DBG] from='mgr.14144 172.21.15.40:0/516882453' entity='mgr.smithi040.ogcprl' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-11-17T05:21:26.254 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:59 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:20:58.549722+0000 mon.smithi040 (mon.0) 74 : audit [DBG] from='mgr.14144 172.21.15.40:0/516882453' entity='mgr.smithi040.ogcprl' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-11-17T05:21:26.254 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:59 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:20:58.566473+0000 mon.smithi040 (mon.0) 75 : audit [INF] from='mgr.14144 172.21.15.40:0/516882453' entity='mgr.smithi040.ogcprl' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi040.ogcprl/mirror_snapshot_schedule"}]: dispatch 2023-11-17T05:21:26.254 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:20:59 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:20:58.569715+0000 mon.smithi040 (mon.0) 76 : audit [INF] from='mgr.14144 172.21.15.40:0/516882453' entity='mgr.smithi040.ogcprl' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi040.ogcprl/trash_purge_schedule"}]: dispatch 2023-11-17T05:21:26.254 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:21:00 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:20:59.427330+0000 mon.smithi040 (mon.0) 77 : cluster [DBG] mgrmap e11: smithi040.ogcprl(active, since 1.06641s) 2023-11-17T05:21:26.254 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:21:00 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cephadm 2023-11-17T05:20:59.428747+0000 mgr.smithi040.ogcprl (mgr.14144) 2 : cephadm [INF] Saving service prometheus spec with placement count:1 2023-11-17T05:21:26.254 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:21:00 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:20:59.431361+0000 mon.smithi040 (mon.0) 78 : audit [INF] from='mgr.14144 172.21.15.40:0/516882453' entity='mgr.smithi040.ogcprl' 2023-11-17T05:21:26.255 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:21:00 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:21:00.367259+0000 mon.smithi040 (mon.0) 79 : audit [INF] from='mgr.14144 172.21.15.40:0/516882453' entity='mgr.smithi040.ogcprl' 2023-11-17T05:21:26.255 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:21:01 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:21:00.364228+0000 mgr.smithi040.ogcprl (mgr.14144) 3 : audit [DBG] from='client.14152 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "target": ["mon-mgr", ""]}]: dispatch 2023-11-17T05:21:26.255 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:21:01 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cephadm 2023-11-17T05:21:00.364801+0000 mgr.smithi040.ogcprl (mgr.14144) 4 : cephadm [INF] Saving service grafana spec with placement count:1 2023-11-17T05:21:26.255 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:21:01 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:21:01.344744+0000 mon.smithi040 (mon.0) 80 : audit [INF] from='mgr.14144 172.21.15.40:0/516882453' entity='mgr.smithi040.ogcprl' 2023-11-17T05:21:26.255 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:21:01 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:21:01.368812+0000 mon.smithi040 (mon.0) 81 : cluster [DBG] mgrmap e12: smithi040.ogcprl(active, since 3s) 2023-11-17T05:21:26.255 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:21:02 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:21:01.341546+0000 mgr.smithi040.ogcprl (mgr.14144) 5 : audit [DBG] from='client.14154 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "node-exporter", "target": ["mon-mgr", ""]}]: dispatch 2023-11-17T05:21:26.255 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:21:02 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cephadm 2023-11-17T05:21:01.342280+0000 mgr.smithi040.ogcprl (mgr.14144) 6 : cephadm [INF] Saving service node-exporter spec with placement * 2023-11-17T05:21:26.255 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:21:02 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:21:02.310782+0000 mon.smithi040 (mon.0) 82 : audit [INF] from='mgr.14144 172.21.15.40:0/516882453' entity='mgr.smithi040.ogcprl' 2023-11-17T05:21:26.255 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:21:03 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:21:02.307820+0000 mgr.smithi040.ogcprl (mgr.14144) 7 : audit [DBG] from='client.14156 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "target": ["mon-mgr", ""]}]: dispatch 2023-11-17T05:21:26.256 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:21:03 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cephadm 2023-11-17T05:21:02.308372+0000 mgr.smithi040.ogcprl (mgr.14144) 8 : cephadm [INF] Saving service alertmanager spec with placement count:1 2023-11-17T05:21:26.256 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:21:03 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:21:03.204287+0000 mon.smithi040 (mon.0) 83 : audit [INF] from='client.? 172.21.15.40:0/1383780309' entity='client.admin' 2023-11-17T05:21:26.256 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:21:05 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:21:04.129521+0000 mon.smithi040 (mon.0) 84 : audit [INF] from='client.? 172.21.15.40:0/2511549213' entity='client.admin' 2023-11-17T05:21:26.256 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:21:05 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:21:05.084707+0000 mon.smithi040 (mon.0) 85 : audit [INF] from='client.? 172.21.15.40:0/4006425044' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "dashboard"}]: dispatch 2023-11-17T05:21:26.256 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:21:06 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:21:05.395523+0000 mon.smithi040 (mon.0) 86 : audit [INF] from='client.? 172.21.15.40:0/4006425044' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "dashboard"}]': finished 2023-11-17T05:21:26.256 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:21:06 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:21:05.395610+0000 mon.smithi040 (mon.0) 87 : cluster [DBG] mgrmap e13: smithi040.ogcprl(active, since 7s) 2023-11-17T05:21:26.256 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:21:07 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:21:06.533576+0000 mon.smithi040 (mon.0) 88 : audit [DBG] from='client.? 172.21.15.40:0/3630407038' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2023-11-17T05:21:26.256 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:21:11 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:21:11.097450+0000 mon.smithi040 (mon.0) 89 : cluster [INF] Active manager daemon smithi040.ogcprl restarted 2023-11-17T05:21:26.256 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:21:11 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:21:11.097896+0000 mon.smithi040 (mon.0) 90 : cluster [INF] Activating manager daemon smithi040.ogcprl 2023-11-17T05:21:26.256 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:21:11 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:21:11.100177+0000 mon.smithi040 (mon.0) 91 : cluster [DBG] osdmap e4: 0 total, 0 up, 0 in 2023-11-17T05:21:26.256 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:21:12 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:21:11.151489+0000 mon.smithi040 (mon.0) 92 : cluster [DBG] mgrmap e14: smithi040.ogcprl(active, starting, since 0.0536834s) 2023-11-17T05:21:26.256 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:21:12 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:21:11.154466+0000 mon.smithi040 (mon.0) 93 : audit [DBG] from='mgr.14164 172.21.15.40:0/2203032451' entity='mgr.smithi040.ogcprl' cmd=[{"prefix": "mon metadata", "id": "smithi040"}]: dispatch 2023-11-17T05:21:26.256 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:21:12 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:21:11.155209+0000 mon.smithi040 (mon.0) 94 : audit [DBG] from='mgr.14164 172.21.15.40:0/2203032451' entity='mgr.smithi040.ogcprl' cmd=[{"prefix": "mgr metadata", "who": "smithi040.ogcprl", "id": "smithi040.ogcprl"}]: dispatch 2023-11-17T05:21:26.257 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:21:12 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:21:11.156256+0000 mon.smithi040 (mon.0) 95 : audit [DBG] from='mgr.14164 172.21.15.40:0/2203032451' entity='mgr.smithi040.ogcprl' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-11-17T05:21:26.257 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:21:12 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:21:11.156428+0000 mon.smithi040 (mon.0) 96 : audit [DBG] from='mgr.14164 172.21.15.40:0/2203032451' entity='mgr.smithi040.ogcprl' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-11-17T05:21:26.257 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:21:12 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:21:11.156591+0000 mon.smithi040 (mon.0) 97 : audit [DBG] from='mgr.14164 172.21.15.40:0/2203032451' entity='mgr.smithi040.ogcprl' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-11-17T05:21:26.257 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:21:12 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:21:11.170208+0000 mon.smithi040 (mon.0) 98 : cluster [INF] Manager daemon smithi040.ogcprl is now available 2023-11-17T05:21:26.257 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:21:12 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:21:11.285060+0000 mon.smithi040 (mon.0) 99 : audit [DBG] from='mgr.14164 172.21.15.40:0/2203032451' entity='mgr.smithi040.ogcprl' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-11-17T05:21:26.257 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:21:12 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:21:11.287866+0000 mon.smithi040 (mon.0) 100 : audit [DBG] from='mgr.14164 172.21.15.40:0/2203032451' entity='mgr.smithi040.ogcprl' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-11-17T05:21:26.257 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:21:12 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:21:11.305159+0000 mon.smithi040 (mon.0) 101 : audit [INF] from='mgr.14164 172.21.15.40:0/2203032451' entity='mgr.smithi040.ogcprl' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi040.ogcprl/mirror_snapshot_schedule"}]: dispatch 2023-11-17T05:21:26.257 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:21:12 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:21:11.308487+0000 mon.smithi040 (mon.0) 102 : audit [INF] from='mgr.14164 172.21.15.40:0/2203032451' entity='mgr.smithi040.ogcprl' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi040.ogcprl/trash_purge_schedule"}]: dispatch 2023-11-17T05:21:26.257 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:21:13 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:21:12.157160+0000 mgr.smithi040.ogcprl (mgr.14164) 1 : audit [DBG] from='client.14168 -' entity='client.admin' cmd=[{"prefix": "get_command_descriptions"}]: dispatch 2023-11-17T05:21:26.257 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:21:13 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:21:12.157513+0000 mon.smithi040 (mon.0) 103 : cluster [DBG] mgrmap e15: smithi040.ogcprl(active, since 1.05971s) 2023-11-17T05:21:26.257 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:21:13 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:21:12.165160+0000 mgr.smithi040.ogcprl (mgr.14164) 2 : audit [DBG] from='client.14168 -' entity='client.admin' cmd=[{"prefix": "mgr_status"}]: dispatch 2023-11-17T05:21:26.257 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:21:14 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:21:13.145041+0000 mgr.smithi040.ogcprl (mgr.14164) 3 : audit [DBG] from='client.14174 -' entity='client.admin' cmd=[{"prefix": "dashboard create-self-signed-cert", "target": ["mon-mgr", ""]}]: dispatch 2023-11-17T05:21:26.257 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:21:14 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:21:13.164453+0000 mon.smithi040 (mon.0) 104 : cluster [DBG] mgrmap e16: smithi040.ogcprl(active, since 2s) 2023-11-17T05:21:26.258 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:21:14 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:21:13.179393+0000 mon.smithi040 (mon.0) 105 : audit [INF] from='mgr.14164 172.21.15.40:0/2203032451' entity='mgr.smithi040.ogcprl' 2023-11-17T05:21:26.258 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:21:14 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:21:13.182258+0000 mon.smithi040 (mon.0) 106 : audit [INF] from='mgr.14164 172.21.15.40:0/2203032451' entity='mgr.smithi040.ogcprl' 2023-11-17T05:21:26.258 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:21:15 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:21:14.181526+0000 mgr.smithi040.ogcprl (mgr.14164) 4 : audit [DBG] from='client.14176 -' entity='client.admin' cmd=[{"prefix": "dashboard ac-user-create", "username": "admin", "rolename": "administrator", "force_password": true, "pwd_update_required": true, "target": ["mon-mgr", ""]}]: dispatch 2023-11-17T05:21:26.258 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:21:15 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:21:14.432869+0000 mon.smithi040 (mon.0) 107 : audit [INF] from='mgr.14164 172.21.15.40:0/2203032451' entity='mgr.smithi040.ogcprl' 2023-11-17T05:21:26.258 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:21:15 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:21:15.348560+0000 mon.smithi040 (mon.0) 108 : audit [DBG] from='client.? 172.21.15.40:0/1923721708' entity='client.admin' cmd=[{"prefix": "config get", "who": "mgr", "key": "mgr/dashboard/ssl_server_port"}]: dispatch 2023-11-17T05:21:26.258 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:21:18 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:21:17.187761+0000 mon.smithi040 (mon.0) 109 : cluster [DBG] mgrmap e17: smithi040.ogcprl(active, since 6s) 2023-11-17T05:21:26.258 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:21:20 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:21:19.500132+0000 mon.smithi040 (mon.0) 110 : audit [INF] from='mgr.14164 172.21.15.40:0/2203032451' entity='mgr.smithi040.ogcprl' 2023-11-17T05:21:26.258 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:21:20 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:21:19.746490+0000 mon.smithi040 (mon.0) 111 : audit [INF] from='mgr.14164 172.21.15.40:0/2203032451' entity='mgr.smithi040.ogcprl' 2023-11-17T05:21:26.258 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:21:20 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:21:20.422875+0000 mon.smithi040 (mon.0) 112 : audit [INF] from='mgr.14164 172.21.15.40:0/2203032451' entity='mgr.smithi040.ogcprl' 2023-11-17T05:21:26.258 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:21:23 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:21:22.840630+0000 mon.smithi040 (mon.0) 113 : audit [INF] from='mgr.14164 172.21.15.40:0/2203032451' entity='mgr.smithi040.ogcprl' 2023-11-17T05:21:26.258 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:21:23 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:21:23.136220+0000 mon.smithi040 (mon.0) 114 : audit [INF] from='mgr.14164 172.21.15.40:0/2203032451' entity='mgr.smithi040.ogcprl' 2023-11-17T05:21:26.258 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:21:23 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:21:23.139347+0000 mon.smithi040 (mon.0) 115 : audit [INF] from='mgr.14164 172.21.15.40:0/2203032451' entity='mgr.smithi040.ogcprl' 2023-11-17T05:21:26.259 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:21:23 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cephadm 2023-11-17T05:21:23.146341+0000 mgr.smithi040.ogcprl (mgr.14164) 5 : cephadm [INF] Deploying daemon alertmanager.smithi040 on smithi040 2023-11-17T05:21:26.261 INFO:teuthology.orchestra.run.smithi040.stderr:mgr epoch 5 is available 2023-11-17T05:21:26.261 INFO:teuthology.orchestra.run.smithi040.stderr:Setting orchestrator backend to cephadm... 2023-11-17T05:21:26.261 INFO:teuthology.orchestra.run.smithi040.stderr:Running command: /bin/podman run --rm --ipc=host --no-hosts --net=host --entrypoint /usr/bin/ceph --init -e CONTAINER_IMAGE=docker.io/ceph/ceph:v16.2.4 -e NODE_NAME=smithi040 -e CEPH_USE_RANDOM_NONCE=1 -v /var/log/ceph/ed6470e2-8508-11ee-8dbe-212e2dc638e7:/var/log/ceph:z -v /tmp/ceph-tmpzx79_7_h:/etc/ceph/ceph.client.admin.keyring:z -v /tmp/ceph-tmpru9tjirw:/etc/ceph/ceph.conf:z docker.io/ceph/ceph:v16.2.4 orch set backend cephadm 2023-11-17T05:21:26.261 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:38.916+0000 7f785783f700 1 Processor -- start 2023-11-17T05:21:26.261 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:38.916+0000 7f785783f700 1 -- start start 2023-11-17T05:21:26.262 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:38.916+0000 7f785783f700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f785006fc90 0x7f7850100be0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 rx=0 tx=0).connect 2023-11-17T05:21:26.262 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:38.916+0000 7f785783f700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f78500fc660 con 0x7f785006fc90 2023-11-17T05:21:26.262 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:38.917+0000 7f78555db700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f785006fc90 0x7f7850100be0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:21:26.262 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:38.917+0000 7f78555db700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f785006fc90 0x7f7850100be0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:51080/0 (socket says 172.21.15.40:51080) 2023-11-17T05:21:26.262 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:38.917+0000 7f78555db700 1 -- 172.21.15.40:0/963036069 learned_addr learned my addr 172.21.15.40:0/963036069 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-11-17T05:21:26.262 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:38.917+0000 7f78555db700 1 -- 172.21.15.40:0/963036069 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f78501010d0 con 0x7f785006fc90 2023-11-17T05:21:26.262 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:38.917+0000 7f78555db700 1 --2- 172.21.15.40:0/963036069 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f785006fc90 0x7f7850100be0 secure :-1 s=READY pgs=31 cs=0 l=1 rev1=1 rx=0x7f784c00b5f0 tx=0x7f784c006ee0).ready entity=mon.0 client_cookie=a091880962b09f7e server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:21:26.262 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:38.917+0000 7f783ffff700 1 -- 172.21.15.40:0/963036069 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f784c00d600 con 0x7f785006fc90 2023-11-17T05:21:26.262 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:38.918+0000 7f783ffff700 1 -- 172.21.15.40:0/963036069 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(25 keys) v1 ==== 1010+0+0 (secure 0 0 0) 0x7f784c00dbd0 con 0x7f785006fc90 2023-11-17T05:21:26.262 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:38.918+0000 7f783ffff700 1 -- 172.21.15.40:0/963036069 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f784c011c00 con 0x7f785006fc90 2023-11-17T05:21:26.262 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:38.918+0000 7f785783f700 1 -- 172.21.15.40:0/963036069 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f785006fc90 msgr2=0x7f7850100be0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:21:26.262 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:38.918+0000 7f785783f700 1 --2- 172.21.15.40:0/963036069 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f785006fc90 0x7f7850100be0 secure :-1 s=READY pgs=31 cs=0 l=1 rev1=1 rx=0x7f784c00b5f0 tx=0x7f784c006ee0).stop 2023-11-17T05:21:26.263 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:38.918+0000 7f785783f700 1 -- 172.21.15.40:0/963036069 shutdown_connections 2023-11-17T05:21:26.263 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:38.918+0000 7f785783f700 1 --2- 172.21.15.40:0/963036069 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f785006fc90 0x7f7850100be0 unknown :-1 s=CLOSED pgs=31 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:21:26.263 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:38.918+0000 7f785783f700 1 -- 172.21.15.40:0/963036069 >> 172.21.15.40:0/963036069 conn(0x7f78500f6280 msgr2=0x7f78500f86a0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:21:26.263 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:38.918+0000 7f785783f700 1 -- 172.21.15.40:0/963036069 shutdown_connections 2023-11-17T05:21:26.263 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:38.918+0000 7f785783f700 1 -- 172.21.15.40:0/963036069 wait complete. 2023-11-17T05:21:26.263 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:38.919+0000 7f785783f700 1 Processor -- start 2023-11-17T05:21:26.263 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:38.919+0000 7f785783f700 1 -- start start 2023-11-17T05:21:26.263 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:38.920+0000 7f785783f700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f785006fc90 0x7f7850110fb0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:21:26.263 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:38.920+0000 7f785783f700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f78501114a0 con 0x7f785006fc90 2023-11-17T05:21:26.263 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:38.920+0000 7f78555db700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f785006fc90 0x7f7850110fb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:21:26.263 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:38.920+0000 7f78555db700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f785006fc90 0x7f7850110fb0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:51084/0 (socket says 172.21.15.40:51084) 2023-11-17T05:21:26.263 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:38.920+0000 7f78555db700 1 -- 172.21.15.40:0/3486887982 learned_addr learned my addr 172.21.15.40:0/3486887982 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-11-17T05:21:26.263 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:38.920+0000 7f78555db700 1 -- 172.21.15.40:0/3486887982 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f784c00b200 con 0x7f785006fc90 2023-11-17T05:21:26.264 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:38.920+0000 7f78555db700 1 --2- 172.21.15.40:0/3486887982 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f785006fc90 0x7f7850110fb0 secure :-1 s=READY pgs=32 cs=0 l=1 rev1=1 rx=0x7f784c00e040 tx=0x7f784c006ee0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:21:26.264 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:38.920+0000 7f783dffb700 1 -- 172.21.15.40:0/3486887982 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f784c01b410 con 0x7f785006fc90 2023-11-17T05:21:26.264 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:38.920+0000 7f783dffb700 1 -- 172.21.15.40:0/3486887982 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(25 keys) v1 ==== 1010+0+0 (secure 0 0 0) 0x7f784c01b9e0 con 0x7f785006fc90 2023-11-17T05:21:26.264 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:38.920+0000 7f785783f700 1 -- 172.21.15.40:0/3486887982 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f78501116a0 con 0x7f785006fc90 2023-11-17T05:21:26.264 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:38.920+0000 7f783dffb700 1 -- 172.21.15.40:0/3486887982 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f784c01a770 con 0x7f785006fc90 2023-11-17T05:21:26.264 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:38.921+0000 7f785783f700 1 -- 172.21.15.40:0/3486887982 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f7850111ab0 con 0x7f785006fc90 2023-11-17T05:21:26.264 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:38.921+0000 7f783dffb700 1 -- 172.21.15.40:0/3486887982 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 7) v1 ==== 36040+0+0 (secure 0 0 0) 0x7f784c00f070 con 0x7f785006fc90 2023-11-17T05:21:26.264 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:38.922+0000 7f783dffb700 1 --2- 172.21.15.40:0/3486887982 >> [v2:172.21.15.40:6800/929662169,v1:172.21.15.40:6801/929662169] conn(0x7f783802fcf0 0x7f78380321b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:21:26.264 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:38.922+0000 7f783dffb700 1 -- 172.21.15.40:0/3486887982 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(2..2 src has 1..2) v4 ==== 926+0+0 (secure 0 0 0) 0x7f784c040f90 con 0x7f785006fc90 2023-11-17T05:21:26.264 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:38.922+0000 7f7854dda700 1 --2- 172.21.15.40:0/3486887982 >> [v2:172.21.15.40:6800/929662169,v1:172.21.15.40:6801/929662169] conn(0x7f783802fcf0 0x7f78380321b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:21:26.264 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:38.922+0000 7f785783f700 1 -- 172.21.15.40:0/3486887982 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f7850059100 con 0x7f785006fc90 2023-11-17T05:21:26.264 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:38.922+0000 7f7854dda700 1 --2- 172.21.15.40:0/3486887982 >> [v2:172.21.15.40:6800/929662169,v1:172.21.15.40:6801/929662169] conn(0x7f783802fcf0 0x7f78380321b0 secure :-1 s=READY pgs=6 cs=0 l=1 rev1=1 rx=0x7f78400096a0 tx=0x7f7840006b40).ready entity=mgr.14116 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:21:26.264 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:38.927+0000 7f783dffb700 1 -- 172.21.15.40:0/3486887982 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+152212 (secure 0 0 0) 0x7f784c016030 con 0x7f785006fc90 2023-11-17T05:21:26.265 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:39.125+0000 7f785783f700 1 -- 172.21.15.40:0/3486887982 --> [v2:172.21.15.40:6800/929662169,v1:172.21.15.40:6801/929662169] -- mgr_command(tid 0: {"prefix": "orch set backend", "module_name": "cephadm", "target": ["mon-mgr", ""]}) v1 -- 0x7f78500ff7f0 con 0x7f783802fcf0 2023-11-17T05:21:26.265 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:39.131+0000 7f783dffb700 1 -- 172.21.15.40:0/3486887982 <== mgr.14116 v2:172.21.15.40:6800/929662169 1 ==== mgr_command_reply(tid 0: 0 ) v1 ==== 8+0+0 (secure 0 0 0) 0x7f78500ff7f0 con 0x7f783802fcf0 2023-11-17T05:21:26.265 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:39.133+0000 7f785783f700 1 -- 172.21.15.40:0/3486887982 >> [v2:172.21.15.40:6800/929662169,v1:172.21.15.40:6801/929662169] conn(0x7f783802fcf0 msgr2=0x7f78380321b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:21:26.275 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:39.133+0000 7f785783f700 1 --2- 172.21.15.40:0/3486887982 >> [v2:172.21.15.40:6800/929662169,v1:172.21.15.40:6801/929662169] conn(0x7f783802fcf0 0x7f78380321b0 secure :-1 s=READY pgs=6 cs=0 l=1 rev1=1 rx=0x7f78400096a0 tx=0x7f7840006b40).stop 2023-11-17T05:21:26.275 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:39.133+0000 7f785783f700 1 -- 172.21.15.40:0/3486887982 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f785006fc90 msgr2=0x7f7850110fb0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:21:26.276 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:39.133+0000 7f785783f700 1 --2- 172.21.15.40:0/3486887982 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f785006fc90 0x7f7850110fb0 secure :-1 s=READY pgs=32 cs=0 l=1 rev1=1 rx=0x7f784c00e040 tx=0x7f784c006ee0).stop 2023-11-17T05:21:26.276 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:39.133+0000 7f785783f700 1 -- 172.21.15.40:0/3486887982 shutdown_connections 2023-11-17T05:21:26.276 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:39.133+0000 7f785783f700 1 --2- 172.21.15.40:0/3486887982 >> [v2:172.21.15.40:6800/929662169,v1:172.21.15.40:6801/929662169] conn(0x7f783802fcf0 0x7f78380321b0 unknown :-1 s=CLOSED pgs=6 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:21:26.276 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:39.133+0000 7f785783f700 1 --2- 172.21.15.40:0/3486887982 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f785006fc90 0x7f7850110fb0 unknown :-1 s=CLOSED pgs=32 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:21:26.276 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:39.133+0000 7f785783f700 1 -- 172.21.15.40:0/3486887982 >> 172.21.15.40:0/3486887982 conn(0x7f78500f6280 msgr2=0x7f78500f7e40 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:21:26.276 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:39.133+0000 7f785783f700 1 -- 172.21.15.40:0/3486887982 shutdown_connections 2023-11-17T05:21:26.276 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:39.133+0000 7f785783f700 1 -- 172.21.15.40:0/3486887982 wait complete. 2023-11-17T05:21:26.276 INFO:teuthology.orchestra.run.smithi040.stderr:Running command: /bin/podman run --rm --ipc=host --no-hosts --net=host --entrypoint /usr/bin/ceph --init -e CONTAINER_IMAGE=docker.io/ceph/ceph:v16.2.4 -e NODE_NAME=smithi040 -e CEPH_USE_RANDOM_NONCE=1 -v /var/log/ceph/ed6470e2-8508-11ee-8dbe-212e2dc638e7:/var/log/ceph:z -v /tmp/ceph-tmpzx79_7_h:/etc/ceph/ceph.client.admin.keyring:z -v /tmp/ceph-tmpru9tjirw:/etc/ceph/ceph.conf:z docker.io/ceph/ceph:v16.2.4 cephadm set-user root 2023-11-17T05:21:26.276 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:39.889+0000 7f14ef59e700 1 Processor -- start 2023-11-17T05:21:26.276 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:39.889+0000 7f14ef59e700 1 -- start start 2023-11-17T05:21:26.276 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:39.890+0000 7f14ef59e700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f14f00fa8f0 0x7f14f00fad10 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 rx=0 tx=0).connect 2023-11-17T05:21:26.276 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:39.890+0000 7f14ef59e700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f14f00fb200 con 0x7f14f00fa8f0 2023-11-17T05:21:26.276 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:39.890+0000 7f14ee59c700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f14f00fa8f0 0x7f14f00fad10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:21:26.277 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:39.890+0000 7f14ee59c700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f14f00fa8f0 0x7f14f00fad10 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:51086/0 (socket says 172.21.15.40:51086) 2023-11-17T05:21:26.277 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:39.890+0000 7f14ee59c700 1 -- 172.21.15.40:0/3758823852 learned_addr learned my addr 172.21.15.40:0/3758823852 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-11-17T05:21:26.277 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:39.891+0000 7f14ee59c700 1 -- 172.21.15.40:0/3758823852 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f14f00fb340 con 0x7f14f00fa8f0 2023-11-17T05:21:26.277 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:39.891+0000 7f14ee59c700 1 --2- 172.21.15.40:0/3758823852 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f14f00fa8f0 0x7f14f00fad10 secure :-1 s=READY pgs=33 cs=0 l=1 rev1=1 rx=0x7f14e000b870 tx=0x7f14e0006f70).ready entity=mon.0 client_cookie=192f83fa9845b165 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:21:26.277 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:39.891+0000 7f14ed59a700 1 -- 172.21.15.40:0/3758823852 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f14e000da20 con 0x7f14f00fa8f0 2023-11-17T05:21:26.277 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:39.891+0000 7f14ed59a700 1 -- 172.21.15.40:0/3758823852 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(25 keys) v1 ==== 1010+0+0 (secure 0 0 0) 0x7f14e0010470 con 0x7f14f00fa8f0 2023-11-17T05:21:26.277 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:39.891+0000 7f14ed59a700 1 -- 172.21.15.40:0/3758823852 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f14e000f5b0 con 0x7f14f00fa8f0 2023-11-17T05:21:26.277 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:39.892+0000 7f14ef59e700 1 -- 172.21.15.40:0/3758823852 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f14f00fa8f0 msgr2=0x7f14f00fad10 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:21:26.277 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:39.892+0000 7f14ef59e700 1 --2- 172.21.15.40:0/3758823852 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f14f00fa8f0 0x7f14f00fad10 secure :-1 s=READY pgs=33 cs=0 l=1 rev1=1 rx=0x7f14e000b870 tx=0x7f14e0006f70).stop 2023-11-17T05:21:26.277 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:39.892+0000 7f14ef59e700 1 -- 172.21.15.40:0/3758823852 shutdown_connections 2023-11-17T05:21:26.277 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:39.892+0000 7f14ef59e700 1 --2- 172.21.15.40:0/3758823852 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f14f00fa8f0 0x7f14f00fad10 unknown :-1 s=CLOSED pgs=33 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:21:26.277 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:39.892+0000 7f14ef59e700 1 -- 172.21.15.40:0/3758823852 >> 172.21.15.40:0/3758823852 conn(0x7f14f00f6130 msgr2=0x7f14f00f8550 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:21:26.277 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:39.892+0000 7f14ef59e700 1 -- 172.21.15.40:0/3758823852 shutdown_connections 2023-11-17T05:21:26.277 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:39.892+0000 7f14ef59e700 1 -- 172.21.15.40:0/3758823852 wait complete. 2023-11-17T05:21:26.278 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:39.892+0000 7f14ef59e700 1 Processor -- start 2023-11-17T05:21:26.278 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:39.893+0000 7f14ef59e700 1 -- start start 2023-11-17T05:21:26.278 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:39.893+0000 7f14ef59e700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f14f00fa8f0 0x7f14f01084d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:21:26.278 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:39.893+0000 7f14ef59e700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f14f01089c0 con 0x7f14f00fa8f0 2023-11-17T05:21:26.278 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:39.893+0000 7f14ee59c700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f14f00fa8f0 0x7f14f01084d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:21:26.278 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:39.893+0000 7f14ee59c700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f14f00fa8f0 0x7f14f01084d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:51102/0 (socket says 172.21.15.40:51102) 2023-11-17T05:21:26.278 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:39.893+0000 7f14ee59c700 1 -- 172.21.15.40:0/1069489644 learned_addr learned my addr 172.21.15.40:0/1069489644 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-11-17T05:21:26.278 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:39.893+0000 7f14ee59c700 1 -- 172.21.15.40:0/1069489644 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f14e000b480 con 0x7f14f00fa8f0 2023-11-17T05:21:26.278 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:39.893+0000 7f14ee59c700 1 --2- 172.21.15.40:0/1069489644 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f14f00fa8f0 0x7f14f01084d0 secure :-1 s=READY pgs=34 cs=0 l=1 rev1=1 rx=0x7f14e0006a40 tx=0x7f14e0006f70).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:21:26.278 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:39.893+0000 7f14deffd700 1 -- 172.21.15.40:0/1069489644 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f14e000f3d0 con 0x7f14f00fa8f0 2023-11-17T05:21:26.278 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:39.894+0000 7f14deffd700 1 -- 172.21.15.40:0/1069489644 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(25 keys) v1 ==== 1010+0+0 (secure 0 0 0) 0x7f14e000fef0 con 0x7f14f00fa8f0 2023-11-17T05:21:26.278 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:39.894+0000 7f14ef59e700 1 -- 172.21.15.40:0/1069489644 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f14f0108bc0 con 0x7f14f00fa8f0 2023-11-17T05:21:26.279 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:39.894+0000 7f14deffd700 1 -- 172.21.15.40:0/1069489644 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f14e0019d40 con 0x7f14f00fa8f0 2023-11-17T05:21:26.279 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:39.894+0000 7f14ef59e700 1 -- 172.21.15.40:0/1069489644 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f14f0108fd0 con 0x7f14f00fa8f0 2023-11-17T05:21:26.279 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:39.894+0000 7f14deffd700 1 -- 172.21.15.40:0/1069489644 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 8) v1 ==== 36120+0+0 (secure 0 0 0) 0x7f14e002b060 con 0x7f14f00fa8f0 2023-11-17T05:21:26.279 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:39.895+0000 7f14deffd700 1 --2- 172.21.15.40:0/1069489644 >> [v2:172.21.15.40:6800/929662169,v1:172.21.15.40:6801/929662169] conn(0x7f14d402fd50 0x7f14d4032210 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:21:26.279 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:39.895+0000 7f14deffd700 1 -- 172.21.15.40:0/1069489644 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(2..2 src has 1..2) v4 ==== 926+0+0 (secure 0 0 0) 0x7f14e0044120 con 0x7f14f00fa8f0 2023-11-17T05:21:26.279 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:39.895+0000 7f14edd9b700 1 --2- 172.21.15.40:0/1069489644 >> [v2:172.21.15.40:6800/929662169,v1:172.21.15.40:6801/929662169] conn(0x7f14d402fd50 0x7f14d4032210 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:21:26.279 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:39.895+0000 7f14ef59e700 1 -- 172.21.15.40:0/1069489644 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f14f0059100 con 0x7f14f00fa8f0 2023-11-17T05:21:26.279 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:39.895+0000 7f14edd9b700 1 --2- 172.21.15.40:0/1069489644 >> [v2:172.21.15.40:6800/929662169,v1:172.21.15.40:6801/929662169] conn(0x7f14d402fd50 0x7f14d4032210 secure :-1 s=READY pgs=7 cs=0 l=1 rev1=1 rx=0x7f14e40096a0 tx=0x7f14e4006b40).ready entity=mgr.14116 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:21:26.279 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:39.900+0000 7f14deffd700 1 -- 172.21.15.40:0/1069489644 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+152212 (secure 0 0 0) 0x7f14e0014020 con 0x7f14f00fa8f0 2023-11-17T05:21:26.279 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:40.068+0000 7f14ef59e700 1 -- 172.21.15.40:0/1069489644 --> [v2:172.21.15.40:6800/929662169,v1:172.21.15.40:6801/929662169] -- mgr_command(tid 0: {"prefix": "cephadm set-user", "user": "root", "target": ["mon-mgr", ""]}) v1 -- 0x7f14f01023b0 con 0x7f14d402fd50 2023-11-17T05:21:26.279 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:40.069+0000 7f14deffd700 1 -- 172.21.15.40:0/1069489644 <== mgr.14116 v2:172.21.15.40:6800/929662169 1 ==== mgr_command_reply(tid 0: 0 ) v1 ==== 8+0+16 (secure 0 0 0) 0x7f14f01023b0 con 0x7f14d402fd50 2023-11-17T05:21:26.279 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: value unchanged 2023-11-17T05:21:26.279 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:40.071+0000 7f14ef59e700 1 -- 172.21.15.40:0/1069489644 >> [v2:172.21.15.40:6800/929662169,v1:172.21.15.40:6801/929662169] conn(0x7f14d402fd50 msgr2=0x7f14d4032210 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:21:26.279 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:40.071+0000 7f14ef59e700 1 --2- 172.21.15.40:0/1069489644 >> [v2:172.21.15.40:6800/929662169,v1:172.21.15.40:6801/929662169] conn(0x7f14d402fd50 0x7f14d4032210 secure :-1 s=READY pgs=7 cs=0 l=1 rev1=1 rx=0x7f14e40096a0 tx=0x7f14e4006b40).stop 2023-11-17T05:21:26.280 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:40.071+0000 7f14ef59e700 1 -- 172.21.15.40:0/1069489644 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f14f00fa8f0 msgr2=0x7f14f01084d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:21:26.280 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:40.071+0000 7f14ef59e700 1 --2- 172.21.15.40:0/1069489644 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f14f00fa8f0 0x7f14f01084d0 secure :-1 s=READY pgs=34 cs=0 l=1 rev1=1 rx=0x7f14e0006a40 tx=0x7f14e0006f70).stop 2023-11-17T05:21:26.280 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:40.071+0000 7f14ef59e700 1 -- 172.21.15.40:0/1069489644 shutdown_connections 2023-11-17T05:21:26.280 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:40.071+0000 7f14ef59e700 1 --2- 172.21.15.40:0/1069489644 >> [v2:172.21.15.40:6800/929662169,v1:172.21.15.40:6801/929662169] conn(0x7f14d402fd50 0x7f14d4032210 unknown :-1 s=CLOSED pgs=7 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:21:26.280 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:40.071+0000 7f14ef59e700 1 --2- 172.21.15.40:0/1069489644 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f14f00fa8f0 0x7f14f01084d0 unknown :-1 s=CLOSED pgs=34 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:21:26.280 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:40.071+0000 7f14ef59e700 1 -- 172.21.15.40:0/1069489644 >> 172.21.15.40:0/1069489644 conn(0x7f14f00f6130 msgr2=0x7f14f00ff680 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:21:26.280 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:40.071+0000 7f14ef59e700 1 -- 172.21.15.40:0/1069489644 shutdown_connections 2023-11-17T05:21:26.280 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:40.071+0000 7f14ef59e700 1 -- 172.21.15.40:0/1069489644 wait complete. 2023-11-17T05:21:26.280 INFO:teuthology.orchestra.run.smithi040.stderr:Generating ssh key... 2023-11-17T05:21:26.280 INFO:teuthology.orchestra.run.smithi040.stderr:Running command: /bin/podman run --rm --ipc=host --no-hosts --net=host --entrypoint /usr/bin/ceph --init -e CONTAINER_IMAGE=docker.io/ceph/ceph:v16.2.4 -e NODE_NAME=smithi040 -e CEPH_USE_RANDOM_NONCE=1 -v /var/log/ceph/ed6470e2-8508-11ee-8dbe-212e2dc638e7:/var/log/ceph:z -v /tmp/ceph-tmpzx79_7_h:/etc/ceph/ceph.client.admin.keyring:z -v /tmp/ceph-tmpru9tjirw:/etc/ceph/ceph.conf:z docker.io/ceph/ceph:v16.2.4 cephadm generate-key 2023-11-17T05:21:26.280 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:40.787+0000 7f9df3256700 1 Processor -- start 2023-11-17T05:21:26.280 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:40.787+0000 7f9df3256700 1 -- start start 2023-11-17T05:21:26.280 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:40.788+0000 7f9df3256700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f9dec06fcd0 0x7f9dec0fe7a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 rx=0 tx=0).connect 2023-11-17T05:21:26.281 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:40.788+0000 7f9df3256700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f9dec0fed20 con 0x7f9dec06fcd0 2023-11-17T05:21:26.281 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:40.788+0000 7f9df0ff2700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f9dec06fcd0 0x7f9dec0fe7a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:21:26.281 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:40.788+0000 7f9df0ff2700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f9dec06fcd0 0x7f9dec0fe7a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:51116/0 (socket says 172.21.15.40:51116) 2023-11-17T05:21:26.281 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:40.788+0000 7f9df0ff2700 1 -- 172.21.15.40:0/2094553338 learned_addr learned my addr 172.21.15.40:0/2094553338 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-11-17T05:21:26.281 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:40.788+0000 7f9df0ff2700 1 -- 172.21.15.40:0/2094553338 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f9dec0fee60 con 0x7f9dec06fcd0 2023-11-17T05:21:26.281 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:40.788+0000 7f9df0ff2700 1 --2- 172.21.15.40:0/2094553338 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f9dec06fcd0 0x7f9dec0fe7a0 secure :-1 s=READY pgs=35 cs=0 l=1 rev1=1 rx=0x7f9de000b5f0 tx=0x7f9de0006ee0).ready entity=mon.0 client_cookie=50ea6c8bd9505e11 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:21:26.281 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:40.789+0000 7f9deb7fe700 1 -- 172.21.15.40:0/2094553338 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f9de000d600 con 0x7f9dec06fcd0 2023-11-17T05:21:26.281 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:40.789+0000 7f9deb7fe700 1 -- 172.21.15.40:0/2094553338 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(25 keys) v1 ==== 1010+0+0 (secure 0 0 0) 0x7f9de000dbd0 con 0x7f9dec06fcd0 2023-11-17T05:21:26.281 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:40.789+0000 7f9deb7fe700 1 -- 172.21.15.40:0/2094553338 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f9de0011c00 con 0x7f9dec06fcd0 2023-11-17T05:21:26.281 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:40.789+0000 7f9df3256700 1 -- 172.21.15.40:0/2094553338 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f9dec06fcd0 msgr2=0x7f9dec0fe7a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:21:26.281 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:40.789+0000 7f9df3256700 1 --2- 172.21.15.40:0/2094553338 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f9dec06fcd0 0x7f9dec0fe7a0 secure :-1 s=READY pgs=35 cs=0 l=1 rev1=1 rx=0x7f9de000b5f0 tx=0x7f9de0006ee0).stop 2023-11-17T05:21:26.281 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:40.789+0000 7f9df3256700 1 -- 172.21.15.40:0/2094553338 shutdown_connections 2023-11-17T05:21:26.281 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:40.789+0000 7f9df3256700 1 --2- 172.21.15.40:0/2094553338 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f9dec06fcd0 0x7f9dec0fe7a0 unknown :-1 s=CLOSED pgs=35 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:21:26.282 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:40.789+0000 7f9df3256700 1 -- 172.21.15.40:0/2094553338 >> 172.21.15.40:0/2094553338 conn(0x7f9dec0f6280 msgr2=0x7f9dec0f86a0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:21:26.282 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:40.790+0000 7f9df3256700 1 -- 172.21.15.40:0/2094553338 shutdown_connections 2023-11-17T05:21:26.282 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:40.790+0000 7f9df3256700 1 -- 172.21.15.40:0/2094553338 wait complete. 2023-11-17T05:21:26.282 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:40.790+0000 7f9df3256700 1 Processor -- start 2023-11-17T05:21:26.282 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:40.790+0000 7f9df3256700 1 -- start start 2023-11-17T05:21:26.282 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:40.791+0000 7f9df3256700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f9dec06fcd0 0x7f9dec0fc220 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:21:26.282 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:40.791+0000 7f9df3256700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f9dec0fa6a0 con 0x7f9dec06fcd0 2023-11-17T05:21:26.282 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:40.791+0000 7f9df0ff2700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f9dec06fcd0 0x7f9dec0fc220 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:21:26.282 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:40.791+0000 7f9df0ff2700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f9dec06fcd0 0x7f9dec0fc220 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:51120/0 (socket says 172.21.15.40:51120) 2023-11-17T05:21:26.282 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:40.791+0000 7f9df0ff2700 1 -- 172.21.15.40:0/2653640869 learned_addr learned my addr 172.21.15.40:0/2653640869 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-11-17T05:21:26.282 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:40.791+0000 7f9df0ff2700 1 -- 172.21.15.40:0/2653640869 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f9de000b200 con 0x7f9dec06fcd0 2023-11-17T05:21:26.282 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:40.791+0000 7f9df0ff2700 1 --2- 172.21.15.40:0/2653640869 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f9dec06fcd0 0x7f9dec0fc220 secure :-1 s=READY pgs=36 cs=0 l=1 rev1=1 rx=0x7f9de000e040 tx=0x7f9de0006ee0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:21:26.282 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:40.791+0000 7f9de97fa700 1 -- 172.21.15.40:0/2653640869 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f9de001b410 con 0x7f9dec06fcd0 2023-11-17T05:21:26.283 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:40.791+0000 7f9de97fa700 1 -- 172.21.15.40:0/2653640869 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(25 keys) v1 ==== 1010+0+0 (secure 0 0 0) 0x7f9de001b9e0 con 0x7f9dec06fcd0 2023-11-17T05:21:26.283 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:40.791+0000 7f9df3256700 1 -- 172.21.15.40:0/2653640869 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f9dec0fa840 con 0x7f9dec06fcd0 2023-11-17T05:21:26.283 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:40.791+0000 7f9de97fa700 1 -- 172.21.15.40:0/2653640869 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f9de001a770 con 0x7f9dec06fcd0 2023-11-17T05:21:26.283 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:40.791+0000 7f9df3256700 1 -- 172.21.15.40:0/2653640869 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f9dec0fac50 con 0x7f9dec06fcd0 2023-11-17T05:21:26.283 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:40.792+0000 7f9de97fa700 1 -- 172.21.15.40:0/2653640869 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 8) v1 ==== 36120+0+0 (secure 0 0 0) 0x7f9de000f070 con 0x7f9dec06fcd0 2023-11-17T05:21:26.283 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:40.792+0000 7f9de97fa700 1 --2- 172.21.15.40:0/2653640869 >> [v2:172.21.15.40:6800/929662169,v1:172.21.15.40:6801/929662169] conn(0x7f9dd002fdb0 0x7f9dd0032270 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:21:26.283 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:40.792+0000 7f9df3256700 1 -- 172.21.15.40:0/2653640869 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f9dec045ad0 con 0x7f9dec06fcd0 2023-11-17T05:21:26.283 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:40.793+0000 7f9de97fa700 1 -- 172.21.15.40:0/2653640869 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(2..2 src has 1..2) v4 ==== 926+0+0 (secure 0 0 0) 0x7f9de00410c0 con 0x7f9dec06fcd0 2023-11-17T05:21:26.283 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:40.793+0000 7f9debfff700 1 --2- 172.21.15.40:0/2653640869 >> [v2:172.21.15.40:6800/929662169,v1:172.21.15.40:6801/929662169] conn(0x7f9dd002fdb0 0x7f9dd0032270 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:21:26.283 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:40.793+0000 7f9debfff700 1 --2- 172.21.15.40:0/2653640869 >> [v2:172.21.15.40:6800/929662169,v1:172.21.15.40:6801/929662169] conn(0x7f9dd002fdb0 0x7f9dd0032270 secure :-1 s=READY pgs=8 cs=0 l=1 rev1=1 rx=0x7f9dd80096a0 tx=0x7f9dd8006b40).ready entity=mgr.14116 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:21:26.283 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:40.797+0000 7f9de97fa700 1 -- 172.21.15.40:0/2653640869 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+152212 (secure 0 0 0) 0x7f9de0016020 con 0x7f9dec06fcd0 2023-11-17T05:21:26.283 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:40.963+0000 7f9df3256700 1 -- 172.21.15.40:0/2653640869 --> [v2:172.21.15.40:6800/929662169,v1:172.21.15.40:6801/929662169] -- mgr_command(tid 0: {"prefix": "cephadm generate-key", "target": ["mon-mgr", ""]}) v1 -- 0x7f9dec104960 con 0x7f9dd002fdb0 2023-11-17T05:21:26.283 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:41.825+0000 7f9de97fa700 1 -- 172.21.15.40:0/2653640869 <== mgr.14116 v2:172.21.15.40:6800/929662169 1 ==== mgr_command_reply(tid 0: 0 ) v1 ==== 8+0+0 (secure 0 0 0) 0x7f9dec104960 con 0x7f9dd002fdb0 2023-11-17T05:21:26.284 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:41.827+0000 7f9df3256700 1 -- 172.21.15.40:0/2653640869 >> [v2:172.21.15.40:6800/929662169,v1:172.21.15.40:6801/929662169] conn(0x7f9dd002fdb0 msgr2=0x7f9dd0032270 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:21:26.284 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:41.827+0000 7f9df3256700 1 --2- 172.21.15.40:0/2653640869 >> [v2:172.21.15.40:6800/929662169,v1:172.21.15.40:6801/929662169] conn(0x7f9dd002fdb0 0x7f9dd0032270 secure :-1 s=READY pgs=8 cs=0 l=1 rev1=1 rx=0x7f9dd80096a0 tx=0x7f9dd8006b40).stop 2023-11-17T05:21:26.284 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:41.827+0000 7f9df3256700 1 -- 172.21.15.40:0/2653640869 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f9dec06fcd0 msgr2=0x7f9dec0fc220 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:21:26.284 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:41.827+0000 7f9df3256700 1 --2- 172.21.15.40:0/2653640869 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f9dec06fcd0 0x7f9dec0fc220 secure :-1 s=READY pgs=36 cs=0 l=1 rev1=1 rx=0x7f9de000e040 tx=0x7f9de0006ee0).stop 2023-11-17T05:21:26.284 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:41.827+0000 7f9df3256700 1 -- 172.21.15.40:0/2653640869 shutdown_connections 2023-11-17T05:21:26.285 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:41.827+0000 7f9df3256700 1 --2- 172.21.15.40:0/2653640869 >> [v2:172.21.15.40:6800/929662169,v1:172.21.15.40:6801/929662169] conn(0x7f9dd002fdb0 0x7f9dd0032270 unknown :-1 s=CLOSED pgs=8 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:21:26.285 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:41.827+0000 7f9df3256700 1 --2- 172.21.15.40:0/2653640869 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f9dec06fcd0 0x7f9dec0fc220 unknown :-1 s=CLOSED pgs=36 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:21:26.285 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:41.827+0000 7f9df3256700 1 -- 172.21.15.40:0/2653640869 >> 172.21.15.40:0/2653640869 conn(0x7f9dec0f6280 msgr2=0x7f9dec0f7e20 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:21:26.285 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:41.827+0000 7f9df3256700 1 -- 172.21.15.40:0/2653640869 shutdown_connections 2023-11-17T05:21:26.286 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:41.827+0000 7f9df3256700 1 -- 172.21.15.40:0/2653640869 wait complete. 2023-11-17T05:21:26.286 INFO:teuthology.orchestra.run.smithi040.stderr:Running command: /bin/podman run --rm --ipc=host --no-hosts --net=host --entrypoint /usr/bin/ceph --init -e CONTAINER_IMAGE=docker.io/ceph/ceph:v16.2.4 -e NODE_NAME=smithi040 -e CEPH_USE_RANDOM_NONCE=1 -v /var/log/ceph/ed6470e2-8508-11ee-8dbe-212e2dc638e7:/var/log/ceph:z -v /tmp/ceph-tmpzx79_7_h:/etc/ceph/ceph.client.admin.keyring:z -v /tmp/ceph-tmpru9tjirw:/etc/ceph/ceph.conf:z docker.io/ceph/ceph:v16.2.4 cephadm get-pub-key 2023-11-17T05:21:26.286 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:45.471+0000 7f62b117d700 1 Processor -- start 2023-11-17T05:21:26.286 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:45.471+0000 7f62b117d700 1 -- start start 2023-11-17T05:21:26.286 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:45.471+0000 7f62b117d700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f62ac06fc50 0x7f62ac100c20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 rx=0 tx=0).connect 2023-11-17T05:21:26.286 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:45.471+0000 7f62b117d700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f62ac0fc6a0 con 0x7f62ac06fc50 2023-11-17T05:21:26.286 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:45.471+0000 7f62aad9d700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f62ac06fc50 0x7f62ac100c20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:21:26.286 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:45.471+0000 7f62aad9d700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f62ac06fc50 0x7f62ac100c20 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:38314/0 (socket says 172.21.15.40:38314) 2023-11-17T05:21:26.286 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:45.471+0000 7f62aad9d700 1 -- 172.21.15.40:0/3332367560 learned_addr learned my addr 172.21.15.40:0/3332367560 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-11-17T05:21:26.286 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:45.471+0000 7f62aad9d700 1 -- 172.21.15.40:0/3332367560 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f62ac101110 con 0x7f62ac06fc50 2023-11-17T05:21:26.287 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:45.471+0000 7f62aad9d700 1 --2- 172.21.15.40:0/3332367560 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f62ac06fc50 0x7f62ac100c20 secure :-1 s=READY pgs=37 cs=0 l=1 rev1=1 rx=0x7f629c00b5f0 tx=0x7f629c006ee0).ready entity=mon.0 client_cookie=7a544eaf00a4114f server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:21:26.287 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:45.472+0000 7f62a9d9b700 1 -- 172.21.15.40:0/3332367560 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f629c00d600 con 0x7f62ac06fc50 2023-11-17T05:21:26.287 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:45.472+0000 7f62a9d9b700 1 -- 172.21.15.40:0/3332367560 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(25 keys) v1 ==== 1010+0+0 (secure 0 0 0) 0x7f629c00dbd0 con 0x7f62ac06fc50 2023-11-17T05:21:26.287 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:45.472+0000 7f62a9d9b700 1 -- 172.21.15.40:0/3332367560 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f629c011c00 con 0x7f62ac06fc50 2023-11-17T05:21:26.287 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:45.472+0000 7f62b117d700 1 -- 172.21.15.40:0/3332367560 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f62ac06fc50 msgr2=0x7f62ac100c20 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:21:26.287 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:45.472+0000 7f62b117d700 1 --2- 172.21.15.40:0/3332367560 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f62ac06fc50 0x7f62ac100c20 secure :-1 s=READY pgs=37 cs=0 l=1 rev1=1 rx=0x7f629c00b5f0 tx=0x7f629c006ee0).stop 2023-11-17T05:21:26.287 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:45.472+0000 7f62b117d700 1 -- 172.21.15.40:0/3332367560 shutdown_connections 2023-11-17T05:21:26.287 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:45.472+0000 7f62b117d700 1 --2- 172.21.15.40:0/3332367560 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f62ac06fc50 0x7f62ac100c20 unknown :-1 s=CLOSED pgs=37 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:21:26.287 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:45.472+0000 7f62b117d700 1 -- 172.21.15.40:0/3332367560 >> 172.21.15.40:0/3332367560 conn(0x7f62ac0f6280 msgr2=0x7f62ac0f86a0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:21:26.287 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:45.472+0000 7f62b117d700 1 -- 172.21.15.40:0/3332367560 shutdown_connections 2023-11-17T05:21:26.288 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:45.472+0000 7f62b117d700 1 -- 172.21.15.40:0/3332367560 wait complete. 2023-11-17T05:21:26.288 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:45.473+0000 7f62b117d700 1 Processor -- start 2023-11-17T05:21:26.288 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:45.473+0000 7f62b117d700 1 -- start start 2023-11-17T05:21:26.288 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:45.473+0000 7f62b117d700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f62ac06fc50 0x7f62ac10ccc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:21:26.288 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:45.473+0000 7f62b117d700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f62ac10d1b0 con 0x7f62ac06fc50 2023-11-17T05:21:26.289 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:45.473+0000 7f62aad9d700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f62ac06fc50 0x7f62ac10ccc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:21:26.289 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:45.473+0000 7f62aad9d700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f62ac06fc50 0x7f62ac10ccc0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:38330/0 (socket says 172.21.15.40:38330) 2023-11-17T05:21:26.289 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:45.473+0000 7f62aad9d700 1 -- 172.21.15.40:0/3273602037 learned_addr learned my addr 172.21.15.40:0/3273602037 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-11-17T05:21:26.289 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:45.474+0000 7f62aad9d700 1 -- 172.21.15.40:0/3273602037 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f629c00b200 con 0x7f62ac06fc50 2023-11-17T05:21:26.289 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:45.474+0000 7f62aad9d700 1 --2- 172.21.15.40:0/3273602037 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f62ac06fc50 0x7f62ac10ccc0 secure :-1 s=READY pgs=38 cs=0 l=1 rev1=1 rx=0x7f629c006c70 tx=0x7f629c006ee0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:21:26.289 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:45.474+0000 7f62a37fe700 1 -- 172.21.15.40:0/3273602037 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f629c011690 con 0x7f62ac06fc50 2023-11-17T05:21:26.289 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:45.474+0000 7f62a37fe700 1 -- 172.21.15.40:0/3273602037 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(25 keys) v1 ==== 1010+0+0 (secure 0 0 0) 0x7f629c01b880 con 0x7f62ac06fc50 2023-11-17T05:21:26.289 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:45.474+0000 7f62b117d700 1 -- 172.21.15.40:0/3273602037 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f62ac10d3b0 con 0x7f62ac06fc50 2023-11-17T05:21:26.289 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:45.474+0000 7f62a37fe700 1 -- 172.21.15.40:0/3273602037 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f629c01a650 con 0x7f62ac06fc50 2023-11-17T05:21:26.289 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:45.474+0000 7f62b117d700 1 -- 172.21.15.40:0/3273602037 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f62ac10d7c0 con 0x7f62ac06fc50 2023-11-17T05:21:26.289 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:45.475+0000 7f62a37fe700 1 -- 172.21.15.40:0/3273602037 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 8) v1 ==== 36120+0+0 (secure 0 0 0) 0x7f629c00f070 con 0x7f62ac06fc50 2023-11-17T05:21:26.289 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:45.475+0000 7f62a37fe700 1 --2- 172.21.15.40:0/3273602037 >> [v2:172.21.15.40:6800/929662169,v1:172.21.15.40:6801/929662169] conn(0x7f628c02fdb0 0x7f628c032270 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:21:26.289 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:45.475+0000 7f62b117d700 1 -- 172.21.15.40:0/3273602037 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f62ac045ad0 con 0x7f62ac06fc50 2023-11-17T05:21:26.290 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:45.475+0000 7f62a37fe700 1 -- 172.21.15.40:0/3273602037 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(2..2 src has 1..2) v4 ==== 926+0+0 (secure 0 0 0) 0x7f629c0411e0 con 0x7f62ac06fc50 2023-11-17T05:21:26.290 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:45.475+0000 7f62aa59c700 1 --2- 172.21.15.40:0/3273602037 >> [v2:172.21.15.40:6800/929662169,v1:172.21.15.40:6801/929662169] conn(0x7f628c02fdb0 0x7f628c032270 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:21:26.290 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:45.476+0000 7f62aa59c700 1 --2- 172.21.15.40:0/3273602037 >> [v2:172.21.15.40:6800/929662169,v1:172.21.15.40:6801/929662169] conn(0x7f628c02fdb0 0x7f628c032270 secure :-1 s=READY pgs=9 cs=0 l=1 rev1=1 rx=0x7f62940096a0 tx=0x7f6294006b40).ready entity=mgr.14116 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:21:26.290 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:45.480+0000 7f62a37fe700 1 -- 172.21.15.40:0/3273602037 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+152212 (secure 0 0 0) 0x7f629c016070 con 0x7f62ac06fc50 2023-11-17T05:21:26.291 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:45.650+0000 7f62b117d700 1 -- 172.21.15.40:0/3273602037 --> [v2:172.21.15.40:6800/929662169,v1:172.21.15.40:6801/929662169] -- mgr_command(tid 0: {"prefix": "cephadm get-pub-key", "target": ["mon-mgr", ""]}) v1 -- 0x7f62ac10e020 con 0x7f628c02fdb0 2023-11-17T05:21:26.292 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:45.650+0000 7f62a37fe700 1 -- 172.21.15.40:0/3273602037 <== mgr.14116 v2:172.21.15.40:6800/929662169 1 ==== mgr_command_reply(tid 0: 0 ) v1 ==== 8+0+595 (secure 0 0 0) 0x7f62ac10e020 con 0x7f628c02fdb0 2023-11-17T05:21:26.292 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDL+SdWMF1qjsHjggKNNpIDFxDY6oTIkPdEqTXFMkmwahWZQUyjp7g+1PyiRV6tvkqmLIRshV75Qr9NMWGqY8e9+o0VBppbADFtyq254i71kMbLUcWa9YzDpgOslDiF5PpfCba9Bhh2XoIwwmPScUUP653EzFNllt2VVFiPZOT7UShuA0viW2dm+gQSY+L8IdDfSFD75vgPm8V2WQdsxKwIoGB0LRD0m5MAoP5+XrMxgLiymH5i4d2k2KfyNAsXKcEZeSFCrun43rXcziCK3kiCIMWuiUIEme2Kuu46MLaRZzuw2MYz3Fx4p/5akiQYxl2JDuBpDHCCxOw2ZOoAT7Hzr1jOpViifPKQ1lFCDpPsxjci/kDbAKhetG/T9e2D9R3nAMmqi/H3fVMSoOIneIS+cWOtK+ZIpFwxcmfVI7NlHNNV4Ckg0xaqAfx5kf/PwK+8vzfkbutSNAIGJ1o6i4Y+70usB7wkLU9r26fs/UsEH7Lip4rTSdQhp+ToebHpBsc= ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7 2023-11-17T05:21:26.292 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:45.652+0000 7f62b117d700 1 -- 172.21.15.40:0/3273602037 >> [v2:172.21.15.40:6800/929662169,v1:172.21.15.40:6801/929662169] conn(0x7f628c02fdb0 msgr2=0x7f628c032270 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:21:26.292 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:45.652+0000 7f62b117d700 1 --2- 172.21.15.40:0/3273602037 >> [v2:172.21.15.40:6800/929662169,v1:172.21.15.40:6801/929662169] conn(0x7f628c02fdb0 0x7f628c032270 secure :-1 s=READY pgs=9 cs=0 l=1 rev1=1 rx=0x7f62940096a0 tx=0x7f6294006b40).stop 2023-11-17T05:21:26.292 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:45.652+0000 7f62b117d700 1 -- 172.21.15.40:0/3273602037 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f62ac06fc50 msgr2=0x7f62ac10ccc0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:21:26.292 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:45.652+0000 7f62b117d700 1 --2- 172.21.15.40:0/3273602037 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f62ac06fc50 0x7f62ac10ccc0 secure :-1 s=READY pgs=38 cs=0 l=1 rev1=1 rx=0x7f629c006c70 tx=0x7f629c006ee0).stop 2023-11-17T05:21:26.292 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:45.652+0000 7f62b117d700 1 -- 172.21.15.40:0/3273602037 shutdown_connections 2023-11-17T05:21:26.292 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:45.652+0000 7f62b117d700 1 --2- 172.21.15.40:0/3273602037 >> [v2:172.21.15.40:6800/929662169,v1:172.21.15.40:6801/929662169] conn(0x7f628c02fdb0 0x7f628c032270 unknown :-1 s=CLOSED pgs=9 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:21:26.292 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:45.652+0000 7f62b117d700 1 --2- 172.21.15.40:0/3273602037 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f62ac06fc50 0x7f62ac10ccc0 unknown :-1 s=CLOSED pgs=38 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:21:26.292 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:45.652+0000 7f62b117d700 1 -- 172.21.15.40:0/3273602037 >> 172.21.15.40:0/3273602037 conn(0x7f62ac0f6280 msgr2=0x7f62ac0f7e40 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:21:26.292 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:45.652+0000 7f62b117d700 1 -- 172.21.15.40:0/3273602037 shutdown_connections 2023-11-17T05:21:26.293 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:45.652+0000 7f62b117d700 1 -- 172.21.15.40:0/3273602037 wait complete. 2023-11-17T05:21:26.293 INFO:teuthology.orchestra.run.smithi040.stderr:Wrote public SSH key to /home/ubuntu/cephtest/ceph.pub 2023-11-17T05:21:26.293 INFO:teuthology.orchestra.run.smithi040.stderr:Adding key to root@localhost authorized_keys... 2023-11-17T05:21:26.293 INFO:teuthology.orchestra.run.smithi040.stderr:Adding host smithi040... 2023-11-17T05:21:26.293 INFO:teuthology.orchestra.run.smithi040.stderr:Running command: /bin/podman run --rm --ipc=host --no-hosts --net=host --entrypoint /usr/bin/ceph --init -e CONTAINER_IMAGE=docker.io/ceph/ceph:v16.2.4 -e NODE_NAME=smithi040 -e CEPH_USE_RANDOM_NONCE=1 -v /var/log/ceph/ed6470e2-8508-11ee-8dbe-212e2dc638e7:/var/log/ceph:z -v /tmp/ceph-tmpzx79_7_h:/etc/ceph/ceph.client.admin.keyring:z -v /tmp/ceph-tmpru9tjirw:/etc/ceph/ceph.conf:z docker.io/ceph/ceph:v16.2.4 orch host add smithi040 172.21.15.40 2023-11-17T05:21:26.293 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:47.874+0000 7f3362e7a700 1 Processor -- start 2023-11-17T05:21:26.293 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:47.874+0000 7f3362e7a700 1 -- start start 2023-11-17T05:21:26.293 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:47.875+0000 7f3362e7a700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f335c0faae0 0x7f335c0faf00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 rx=0 tx=0).connect 2023-11-17T05:21:26.293 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:47.875+0000 7f3362e7a700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f335c0fb480 con 0x7f335c0faae0 2023-11-17T05:21:26.293 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:47.875+0000 7f3360c16700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f335c0faae0 0x7f335c0faf00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:21:26.293 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:47.875+0000 7f3360c16700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f335c0faae0 0x7f335c0faf00 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:38344/0 (socket says 172.21.15.40:38344) 2023-11-17T05:21:26.294 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:47.875+0000 7f3360c16700 1 -- 172.21.15.40:0/3853106470 learned_addr learned my addr 172.21.15.40:0/3853106470 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-11-17T05:21:26.294 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:47.875+0000 7f3360c16700 1 -- 172.21.15.40:0/3853106470 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f335c0fb5c0 con 0x7f335c0faae0 2023-11-17T05:21:26.294 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:47.875+0000 7f3360c16700 1 --2- 172.21.15.40:0/3853106470 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f335c0faae0 0x7f335c0faf00 secure :-1 s=READY pgs=39 cs=0 l=1 rev1=1 rx=0x7f335000b5f0 tx=0x7f3350006ee0).ready entity=mon.0 client_cookie=28e0e3e8e5a2c995 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:21:26.294 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:47.875+0000 7f335b7fe700 1 -- 172.21.15.40:0/3853106470 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f335000d600 con 0x7f335c0faae0 2023-11-17T05:21:26.294 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:47.875+0000 7f335b7fe700 1 -- 172.21.15.40:0/3853106470 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(25 keys) v1 ==== 1010+0+0 (secure 0 0 0) 0x7f335000dbd0 con 0x7f335c0faae0 2023-11-17T05:21:26.294 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:47.875+0000 7f335b7fe700 1 -- 172.21.15.40:0/3853106470 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f3350011c00 con 0x7f335c0faae0 2023-11-17T05:21:26.294 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:47.876+0000 7f3362e7a700 1 -- 172.21.15.40:0/3853106470 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f335c0faae0 msgr2=0x7f335c0faf00 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:21:26.294 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:47.876+0000 7f3362e7a700 1 --2- 172.21.15.40:0/3853106470 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f335c0faae0 0x7f335c0faf00 secure :-1 s=READY pgs=39 cs=0 l=1 rev1=1 rx=0x7f335000b5f0 tx=0x7f3350006ee0).stop 2023-11-17T05:21:26.294 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:47.876+0000 7f3362e7a700 1 -- 172.21.15.40:0/3853106470 shutdown_connections 2023-11-17T05:21:26.294 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:47.876+0000 7f3362e7a700 1 --2- 172.21.15.40:0/3853106470 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f335c0faae0 0x7f335c0faf00 unknown :-1 s=CLOSED pgs=39 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:21:26.295 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:47.876+0000 7f3362e7a700 1 -- 172.21.15.40:0/3853106470 >> 172.21.15.40:0/3853106470 conn(0x7f335c0f6280 msgr2=0x7f335c0f86a0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:21:26.295 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:47.876+0000 7f3362e7a700 1 -- 172.21.15.40:0/3853106470 shutdown_connections 2023-11-17T05:21:26.295 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:47.876+0000 7f3362e7a700 1 -- 172.21.15.40:0/3853106470 wait complete. 2023-11-17T05:21:26.295 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:47.877+0000 7f3362e7a700 1 Processor -- start 2023-11-17T05:21:26.295 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:47.877+0000 7f3362e7a700 1 -- start start 2023-11-17T05:21:26.295 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:47.877+0000 7f3362e7a700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f335c0faae0 0x7f335c100df0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:21:26.295 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:47.877+0000 7f3362e7a700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f335c0ff270 con 0x7f335c0faae0 2023-11-17T05:21:26.295 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:47.877+0000 7f3360c16700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f335c0faae0 0x7f335c100df0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:21:26.295 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:47.877+0000 7f3360c16700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f335c0faae0 0x7f335c100df0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:38354/0 (socket says 172.21.15.40:38354) 2023-11-17T05:21:26.295 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:47.877+0000 7f3360c16700 1 -- 172.21.15.40:0/964866914 learned_addr learned my addr 172.21.15.40:0/964866914 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-11-17T05:21:26.295 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:47.878+0000 7f3360c16700 1 -- 172.21.15.40:0/964866914 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f335000b200 con 0x7f335c0faae0 2023-11-17T05:21:26.296 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:47.878+0000 7f3360c16700 1 --2- 172.21.15.40:0/964866914 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f335c0faae0 0x7f335c100df0 secure :-1 s=READY pgs=40 cs=0 l=1 rev1=1 rx=0x7f3350002f40 tx=0x7f3350006ee0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:21:26.296 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:47.878+0000 7f33597fa700 1 -- 172.21.15.40:0/964866914 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f3350011690 con 0x7f335c0faae0 2023-11-17T05:21:26.296 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:47.878+0000 7f3362e7a700 1 -- 172.21.15.40:0/964866914 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f335c0ff410 con 0x7f335c0faae0 2023-11-17T05:21:26.296 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:47.878+0000 7f33597fa700 1 -- 172.21.15.40:0/964866914 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(25 keys) v1 ==== 1010+0+0 (secure 0 0 0) 0x7f335001b880 con 0x7f335c0faae0 2023-11-17T05:21:26.296 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:47.878+0000 7f33597fa700 1 -- 172.21.15.40:0/964866914 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f335001a6b0 con 0x7f335c0faae0 2023-11-17T05:21:26.296 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:47.878+0000 7f3362e7a700 1 -- 172.21.15.40:0/964866914 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f335c0ff820 con 0x7f335c0faae0 2023-11-17T05:21:26.296 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:47.879+0000 7f33597fa700 1 -- 172.21.15.40:0/964866914 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 8) v1 ==== 36120+0+0 (secure 0 0 0) 0x7f335000f070 con 0x7f335c0faae0 2023-11-17T05:21:26.296 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:47.879+0000 7f33597fa700 1 --2- 172.21.15.40:0/964866914 >> [v2:172.21.15.40:6800/929662169,v1:172.21.15.40:6801/929662169] conn(0x7f334002fd50 0x7f3340032210 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:21:26.296 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:47.879+0000 7f3362e7a700 1 -- 172.21.15.40:0/964866914 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f335c045ad0 con 0x7f335c0faae0 2023-11-17T05:21:26.296 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:47.879+0000 7f33597fa700 1 -- 172.21.15.40:0/964866914 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(2..2 src has 1..2) v4 ==== 926+0+0 (secure 0 0 0) 0x7f3350041140 con 0x7f335c0faae0 2023-11-17T05:21:26.296 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:47.879+0000 7f335bfff700 1 --2- 172.21.15.40:0/964866914 >> [v2:172.21.15.40:6800/929662169,v1:172.21.15.40:6801/929662169] conn(0x7f334002fd50 0x7f3340032210 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:21:26.296 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:47.880+0000 7f335bfff700 1 --2- 172.21.15.40:0/964866914 >> [v2:172.21.15.40:6800/929662169,v1:172.21.15.40:6801/929662169] conn(0x7f334002fd50 0x7f3340032210 secure :-1 s=READY pgs=10 cs=0 l=1 rev1=1 rx=0x7f33480096a0 tx=0x7f3348006b40).ready entity=mgr.14116 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:21:26.297 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:47.884+0000 7f33597fa700 1 -- 172.21.15.40:0/964866914 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+152212 (secure 0 0 0) 0x7f3350016070 con 0x7f335c0faae0 2023-11-17T05:21:26.297 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:48.073+0000 7f3362e7a700 1 -- 172.21.15.40:0/964866914 --> [v2:172.21.15.40:6800/929662169,v1:172.21.15.40:6801/929662169] -- mgr_command(tid 0: {"prefix": "orch host add", "hostname": "smithi040", "addr": "172.21.15.40", "target": ["mon-mgr", ""]}) v1 -- 0x7f335c0ffa60 con 0x7f334002fd50 2023-11-17T05:21:26.297 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: Added host 'smithi040' 2023-11-17T05:21:26.297 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:48.738+0000 7f33597fa700 1 -- 172.21.15.40:0/964866914 <== mgr.14116 v2:172.21.15.40:6800/929662169 1 ==== mgr_command_reply(tid 0: 0 ) v1 ==== 8+0+23 (secure 0 0 0) 0x7f335c0ffa60 con 0x7f334002fd50 2023-11-17T05:21:26.297 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:48.740+0000 7f3362e7a700 1 -- 172.21.15.40:0/964866914 >> [v2:172.21.15.40:6800/929662169,v1:172.21.15.40:6801/929662169] conn(0x7f334002fd50 msgr2=0x7f3340032210 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:21:26.297 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:48.740+0000 7f3362e7a700 1 --2- 172.21.15.40:0/964866914 >> [v2:172.21.15.40:6800/929662169,v1:172.21.15.40:6801/929662169] conn(0x7f334002fd50 0x7f3340032210 secure :-1 s=READY pgs=10 cs=0 l=1 rev1=1 rx=0x7f33480096a0 tx=0x7f3348006b40).stop 2023-11-17T05:21:26.297 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:48.741+0000 7f3362e7a700 1 -- 172.21.15.40:0/964866914 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f335c0faae0 msgr2=0x7f335c100df0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:21:26.297 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:48.741+0000 7f3362e7a700 1 --2- 172.21.15.40:0/964866914 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f335c0faae0 0x7f335c100df0 secure :-1 s=READY pgs=40 cs=0 l=1 rev1=1 rx=0x7f3350002f40 tx=0x7f3350006ee0).stop 2023-11-17T05:21:26.297 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:48.741+0000 7f3362e7a700 1 -- 172.21.15.40:0/964866914 shutdown_connections 2023-11-17T05:21:26.297 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:48.741+0000 7f3362e7a700 1 --2- 172.21.15.40:0/964866914 >> [v2:172.21.15.40:6800/929662169,v1:172.21.15.40:6801/929662169] conn(0x7f334002fd50 0x7f3340032210 unknown :-1 s=CLOSED pgs=10 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:21:26.298 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:48.741+0000 7f3362e7a700 1 --2- 172.21.15.40:0/964866914 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f335c0faae0 0x7f335c100df0 unknown :-1 s=CLOSED pgs=40 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:21:26.298 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:48.741+0000 7f3362e7a700 1 -- 172.21.15.40:0/964866914 >> 172.21.15.40:0/964866914 conn(0x7f335c0f6280 msgr2=0x7f335c06f890 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:21:26.298 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:48.741+0000 7f3362e7a700 1 -- 172.21.15.40:0/964866914 shutdown_connections 2023-11-17T05:21:26.298 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:48.741+0000 7f3362e7a700 1 -- 172.21.15.40:0/964866914 wait complete. 2023-11-17T05:21:26.298 INFO:teuthology.orchestra.run.smithi040.stderr:Deploying mon service with default placement... 2023-11-17T05:21:26.298 INFO:teuthology.orchestra.run.smithi040.stderr:Running command: /bin/podman run --rm --ipc=host --no-hosts --net=host --entrypoint /usr/bin/ceph --init -e CONTAINER_IMAGE=docker.io/ceph/ceph:v16.2.4 -e NODE_NAME=smithi040 -e CEPH_USE_RANDOM_NONCE=1 -v /var/log/ceph/ed6470e2-8508-11ee-8dbe-212e2dc638e7:/var/log/ceph:z -v /tmp/ceph-tmpzx79_7_h:/etc/ceph/ceph.client.admin.keyring:z -v /tmp/ceph-tmpru9tjirw:/etc/ceph/ceph.conf:z docker.io/ceph/ceph:v16.2.4 orch apply mon 2023-11-17T05:21:26.299 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:49.428+0000 7f7191b62700 1 Processor -- start 2023-11-17T05:21:26.299 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:49.428+0000 7f7191b62700 1 -- start start 2023-11-17T05:21:26.299 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:49.428+0000 7f7191b62700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f718c06fcd0 0x7f718c0fe7a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 rx=0 tx=0).connect 2023-11-17T05:21:26.299 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:49.428+0000 7f7191b62700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f718c0fed20 con 0x7f718c06fcd0 2023-11-17T05:21:26.299 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:49.429+0000 7f718b7fe700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f718c06fcd0 0x7f718c0fe7a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:21:26.299 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:49.429+0000 7f718b7fe700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f718c06fcd0 0x7f718c0fe7a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:38364/0 (socket says 172.21.15.40:38364) 2023-11-17T05:21:26.299 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:49.429+0000 7f718b7fe700 1 -- 172.21.15.40:0/2278976391 learned_addr learned my addr 172.21.15.40:0/2278976391 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-11-17T05:21:26.300 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:49.429+0000 7f718b7fe700 1 -- 172.21.15.40:0/2278976391 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f718c0fee60 con 0x7f718c06fcd0 2023-11-17T05:21:26.300 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:49.429+0000 7f718b7fe700 1 --2- 172.21.15.40:0/2278976391 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f718c06fcd0 0x7f718c0fe7a0 secure :-1 s=READY pgs=41 cs=0 l=1 rev1=1 rx=0x7f717400b5f0 tx=0x7f7174006ee0).ready entity=mon.0 client_cookie=936c0822e5676ae server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:21:26.300 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:49.429+0000 7f718a7fc700 1 -- 172.21.15.40:0/2278976391 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f717400d600 con 0x7f718c06fcd0 2023-11-17T05:21:26.300 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:49.429+0000 7f718a7fc700 1 -- 172.21.15.40:0/2278976391 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(25 keys) v1 ==== 1010+0+0 (secure 0 0 0) 0x7f717400dbd0 con 0x7f718c06fcd0 2023-11-17T05:21:26.300 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:49.429+0000 7f718a7fc700 1 -- 172.21.15.40:0/2278976391 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f7174011c00 con 0x7f718c06fcd0 2023-11-17T05:21:26.300 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:49.430+0000 7f7191b62700 1 -- 172.21.15.40:0/2278976391 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f718c06fcd0 msgr2=0x7f718c0fe7a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:21:26.300 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:49.430+0000 7f7191b62700 1 --2- 172.21.15.40:0/2278976391 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f718c06fcd0 0x7f718c0fe7a0 secure :-1 s=READY pgs=41 cs=0 l=1 rev1=1 rx=0x7f717400b5f0 tx=0x7f7174006ee0).stop 2023-11-17T05:21:26.300 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:49.430+0000 7f7191b62700 1 -- 172.21.15.40:0/2278976391 shutdown_connections 2023-11-17T05:21:26.301 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:49.430+0000 7f7191b62700 1 --2- 172.21.15.40:0/2278976391 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f718c06fcd0 0x7f718c0fe7a0 unknown :-1 s=CLOSED pgs=41 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:21:26.301 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:49.430+0000 7f7191b62700 1 -- 172.21.15.40:0/2278976391 >> 172.21.15.40:0/2278976391 conn(0x7f718c0f6280 msgr2=0x7f718c0f86a0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:21:26.301 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:49.430+0000 7f7191b62700 1 -- 172.21.15.40:0/2278976391 shutdown_connections 2023-11-17T05:21:26.301 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:49.430+0000 7f7191b62700 1 -- 172.21.15.40:0/2278976391 wait complete. 2023-11-17T05:21:26.301 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:49.430+0000 7f7191b62700 1 Processor -- start 2023-11-17T05:21:26.301 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:49.431+0000 7f7191b62700 1 -- start start 2023-11-17T05:21:26.301 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:49.431+0000 7f7191b62700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f718c06fcd0 0x7f718c0fc220 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:21:26.301 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:49.431+0000 7f7191b62700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f718c0fa6a0 con 0x7f718c06fcd0 2023-11-17T05:21:26.301 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:49.431+0000 7f718b7fe700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f718c06fcd0 0x7f718c0fc220 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:21:26.301 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:49.431+0000 7f718b7fe700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f718c06fcd0 0x7f718c0fc220 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:38380/0 (socket says 172.21.15.40:38380) 2023-11-17T05:21:26.301 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:49.431+0000 7f718b7fe700 1 -- 172.21.15.40:0/1471495276 learned_addr learned my addr 172.21.15.40:0/1471495276 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-11-17T05:21:26.302 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:49.431+0000 7f718b7fe700 1 -- 172.21.15.40:0/1471495276 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f717400b200 con 0x7f718c06fcd0 2023-11-17T05:21:26.302 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:49.432+0000 7f718b7fe700 1 --2- 172.21.15.40:0/1471495276 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f718c06fcd0 0x7f718c0fc220 secure :-1 s=READY pgs=42 cs=0 l=1 rev1=1 rx=0x7f7174000c00 tx=0x7f7174006ee0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:21:26.302 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:49.432+0000 7f7183fff700 1 -- 172.21.15.40:0/1471495276 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f7174011690 con 0x7f718c06fcd0 2023-11-17T05:21:26.302 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:49.432+0000 7f7183fff700 1 -- 172.21.15.40:0/1471495276 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(25 keys) v1 ==== 1010+0+0 (secure 0 0 0) 0x7f717400d9f0 con 0x7f718c06fcd0 2023-11-17T05:21:26.302 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:49.432+0000 7f7191b62700 1 -- 172.21.15.40:0/1471495276 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f718c0fa840 con 0x7f718c06fcd0 2023-11-17T05:21:26.302 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:49.432+0000 7f7183fff700 1 -- 172.21.15.40:0/1471495276 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f717401b710 con 0x7f718c06fcd0 2023-11-17T05:21:26.302 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:49.432+0000 7f7191b62700 1 -- 172.21.15.40:0/1471495276 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f718c0fac50 con 0x7f718c06fcd0 2023-11-17T05:21:26.302 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:49.433+0000 7f7183fff700 1 -- 172.21.15.40:0/1471495276 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 8) v1 ==== 36120+0+0 (secure 0 0 0) 0x7f717400f070 con 0x7f718c06fcd0 2023-11-17T05:21:26.302 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:49.433+0000 7f7191b62700 1 -- 172.21.15.40:0/1471495276 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f718c045ad0 con 0x7f718c06fcd0 2023-11-17T05:21:26.302 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:49.433+0000 7f7183fff700 1 --2- 172.21.15.40:0/1471495276 >> [v2:172.21.15.40:6800/929662169,v1:172.21.15.40:6801/929662169] conn(0x7f716c02fd50 0x7f716c032210 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:21:26.303 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:49.433+0000 7f7183fff700 1 -- 172.21.15.40:0/1471495276 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(2..2 src has 1..2) v4 ==== 926+0+0 (secure 0 0 0) 0x7f7174041160 con 0x7f718c06fcd0 2023-11-17T05:21:26.303 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:49.433+0000 7f718affd700 1 --2- 172.21.15.40:0/1471495276 >> [v2:172.21.15.40:6800/929662169,v1:172.21.15.40:6801/929662169] conn(0x7f716c02fd50 0x7f716c032210 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:21:26.303 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:49.434+0000 7f718affd700 1 --2- 172.21.15.40:0/1471495276 >> [v2:172.21.15.40:6800/929662169,v1:172.21.15.40:6801/929662169] conn(0x7f716c02fd50 0x7f716c032210 secure :-1 s=READY pgs=11 cs=0 l=1 rev1=1 rx=0x7f717c0096a0 tx=0x7f717c006b40).ready entity=mgr.14116 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:21:26.303 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:49.438+0000 7f7183fff700 1 -- 172.21.15.40:0/1471495276 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+152212 (secure 0 0 0) 0x7f71740224f0 con 0x7f718c06fcd0 2023-11-17T05:21:26.303 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:49.632+0000 7f7191b62700 1 -- 172.21.15.40:0/1471495276 --> [v2:172.21.15.40:6800/929662169,v1:172.21.15.40:6801/929662169] -- mgr_command(tid 0: {"prefix": "orch apply", "service_type": "mon", "target": ["mon-mgr", ""]}) v1 -- 0x7f718c0ffd10 con 0x7f716c02fd50 2023-11-17T05:21:26.305 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:49.637+0000 7f7183fff700 1 -- 172.21.15.40:0/1471495276 <== mgr.14116 v2:172.21.15.40:6800/929662169 1 ==== mgr_command_reply(tid 0: 0 ) v1 ==== 8+0+24 (secure 0 0 0) 0x7f718c0ffd10 con 0x7f716c02fd50 2023-11-17T05:21:26.305 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: Scheduled mon update... 2023-11-17T05:21:26.306 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:49.639+0000 7f7191b62700 1 -- 172.21.15.40:0/1471495276 >> [v2:172.21.15.40:6800/929662169,v1:172.21.15.40:6801/929662169] conn(0x7f716c02fd50 msgr2=0x7f716c032210 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:21:26.306 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:49.639+0000 7f7191b62700 1 --2- 172.21.15.40:0/1471495276 >> [v2:172.21.15.40:6800/929662169,v1:172.21.15.40:6801/929662169] conn(0x7f716c02fd50 0x7f716c032210 secure :-1 s=READY pgs=11 cs=0 l=1 rev1=1 rx=0x7f717c0096a0 tx=0x7f717c006b40).stop 2023-11-17T05:21:26.306 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:49.639+0000 7f7191b62700 1 -- 172.21.15.40:0/1471495276 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f718c06fcd0 msgr2=0x7f718c0fc220 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:21:26.306 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:49.639+0000 7f7191b62700 1 --2- 172.21.15.40:0/1471495276 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f718c06fcd0 0x7f718c0fc220 secure :-1 s=READY pgs=42 cs=0 l=1 rev1=1 rx=0x7f7174000c00 tx=0x7f7174006ee0).stop 2023-11-17T05:21:26.306 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:49.639+0000 7f7191b62700 1 -- 172.21.15.40:0/1471495276 shutdown_connections 2023-11-17T05:21:26.306 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:49.639+0000 7f7191b62700 1 --2- 172.21.15.40:0/1471495276 >> [v2:172.21.15.40:6800/929662169,v1:172.21.15.40:6801/929662169] conn(0x7f716c02fd50 0x7f716c032210 unknown :-1 s=CLOSED pgs=11 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:21:26.306 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:49.639+0000 7f7191b62700 1 --2- 172.21.15.40:0/1471495276 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f718c06fcd0 0x7f718c0fc220 unknown :-1 s=CLOSED pgs=42 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:21:26.306 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:49.639+0000 7f7191b62700 1 -- 172.21.15.40:0/1471495276 >> 172.21.15.40:0/1471495276 conn(0x7f718c0f6280 msgr2=0x7f718c0f7e70 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:21:26.306 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:49.639+0000 7f7191b62700 1 -- 172.21.15.40:0/1471495276 shutdown_connections 2023-11-17T05:21:26.306 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:49.639+0000 7f7191b62700 1 -- 172.21.15.40:0/1471495276 wait complete. 2023-11-17T05:21:26.306 INFO:teuthology.orchestra.run.smithi040.stderr:Deploying mgr service with default placement... 2023-11-17T05:21:26.306 INFO:teuthology.orchestra.run.smithi040.stderr:Running command: /bin/podman run --rm --ipc=host --no-hosts --net=host --entrypoint /usr/bin/ceph --init -e CONTAINER_IMAGE=docker.io/ceph/ceph:v16.2.4 -e NODE_NAME=smithi040 -e CEPH_USE_RANDOM_NONCE=1 -v /var/log/ceph/ed6470e2-8508-11ee-8dbe-212e2dc638e7:/var/log/ceph:z -v /tmp/ceph-tmpzx79_7_h:/etc/ceph/ceph.client.admin.keyring:z -v /tmp/ceph-tmpru9tjirw:/etc/ceph/ceph.conf:z docker.io/ceph/ceph:v16.2.4 orch apply mgr 2023-11-17T05:21:26.306 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:50.397+0000 7ff4d89f4700 1 Processor -- start 2023-11-17T05:21:26.307 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:50.397+0000 7ff4d89f4700 1 -- start start 2023-11-17T05:21:26.307 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:50.397+0000 7ff4d89f4700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7ff4d406fc10 0x7ff4d40fe860 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 rx=0 tx=0).connect 2023-11-17T05:21:26.307 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:50.397+0000 7ff4d89f4700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7ff4d40fed50 con 0x7ff4d406fc10 2023-11-17T05:21:26.307 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:50.397+0000 7ff4d259c700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7ff4d406fc10 0x7ff4d40fe860 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:21:26.307 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:50.397+0000 7ff4d259c700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7ff4d406fc10 0x7ff4d40fe860 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:38384/0 (socket says 172.21.15.40:38384) 2023-11-17T05:21:26.307 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:50.397+0000 7ff4d259c700 1 -- 172.21.15.40:0/1035506095 learned_addr learned my addr 172.21.15.40:0/1035506095 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-11-17T05:21:26.308 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:50.398+0000 7ff4d259c700 1 -- 172.21.15.40:0/1035506095 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7ff4d40fee90 con 0x7ff4d406fc10 2023-11-17T05:21:26.308 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:50.398+0000 7ff4d259c700 1 --2- 172.21.15.40:0/1035506095 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7ff4d406fc10 0x7ff4d40fe860 secure :-1 s=READY pgs=43 cs=0 l=1 rev1=1 rx=0x7ff4c400b5f0 tx=0x7ff4c4006ee0).ready entity=mon.0 client_cookie=82c967605e205372 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:21:26.308 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:50.398+0000 7ff4d159a700 1 -- 172.21.15.40:0/1035506095 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7ff4c400d600 con 0x7ff4d406fc10 2023-11-17T05:21:26.308 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:50.398+0000 7ff4d159a700 1 -- 172.21.15.40:0/1035506095 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(25 keys) v1 ==== 1010+0+0 (secure 0 0 0) 0x7ff4c400dbd0 con 0x7ff4d406fc10 2023-11-17T05:21:26.308 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:50.398+0000 7ff4d159a700 1 -- 172.21.15.40:0/1035506095 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7ff4c4011c30 con 0x7ff4d406fc10 2023-11-17T05:21:26.308 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:50.398+0000 7ff4d89f4700 1 -- 172.21.15.40:0/1035506095 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7ff4d406fc10 msgr2=0x7ff4d40fe860 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:21:26.308 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:50.398+0000 7ff4d89f4700 1 --2- 172.21.15.40:0/1035506095 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7ff4d406fc10 0x7ff4d40fe860 secure :-1 s=READY pgs=43 cs=0 l=1 rev1=1 rx=0x7ff4c400b5f0 tx=0x7ff4c4006ee0).stop 2023-11-17T05:21:26.308 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:50.399+0000 7ff4d89f4700 1 -- 172.21.15.40:0/1035506095 shutdown_connections 2023-11-17T05:21:26.308 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:50.399+0000 7ff4d89f4700 1 --2- 172.21.15.40:0/1035506095 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7ff4d406fc10 0x7ff4d40fe860 unknown :-1 s=CLOSED pgs=43 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:21:26.308 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:50.399+0000 7ff4d89f4700 1 -- 172.21.15.40:0/1035506095 >> 172.21.15.40:0/1035506095 conn(0x7ff4d40f6280 msgr2=0x7ff4d40f86a0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:21:26.308 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:50.399+0000 7ff4d89f4700 1 -- 172.21.15.40:0/1035506095 shutdown_connections 2023-11-17T05:21:26.308 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:50.399+0000 7ff4d89f4700 1 -- 172.21.15.40:0/1035506095 wait complete. 2023-11-17T05:21:26.309 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:50.399+0000 7ff4d89f4700 1 Processor -- start 2023-11-17T05:21:26.309 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:50.399+0000 7ff4d89f4700 1 -- start start 2023-11-17T05:21:26.309 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:50.400+0000 7ff4d89f4700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7ff4d406fc10 0x7ff4d40fc2e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:21:26.309 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:50.400+0000 7ff4d89f4700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7ff4d40fa760 con 0x7ff4d406fc10 2023-11-17T05:21:26.310 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:50.400+0000 7ff4d259c700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7ff4d406fc10 0x7ff4d40fc2e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:21:26.310 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:50.400+0000 7ff4d259c700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7ff4d406fc10 0x7ff4d40fc2e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:38400/0 (socket says 172.21.15.40:38400) 2023-11-17T05:21:26.310 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:50.400+0000 7ff4d259c700 1 -- 172.21.15.40:0/3923973603 learned_addr learned my addr 172.21.15.40:0/3923973603 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-11-17T05:21:26.310 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:50.400+0000 7ff4d259c700 1 -- 172.21.15.40:0/3923973603 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7ff4c400b200 con 0x7ff4d406fc10 2023-11-17T05:21:26.310 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:50.400+0000 7ff4d259c700 1 --2- 172.21.15.40:0/3923973603 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7ff4d406fc10 0x7ff4d40fc2e0 secure :-1 s=READY pgs=44 cs=0 l=1 rev1=1 rx=0x7ff4c4011550 tx=0x7ff4c4006ee0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:21:26.310 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:50.400+0000 7ff4c2ffd700 1 -- 172.21.15.40:0/3923973603 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7ff4c401b410 con 0x7ff4d406fc10 2023-11-17T05:21:26.310 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:50.400+0000 7ff4d89f4700 1 -- 172.21.15.40:0/3923973603 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7ff4d40fa900 con 0x7ff4d406fc10 2023-11-17T05:21:26.310 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:50.400+0000 7ff4c2ffd700 1 -- 172.21.15.40:0/3923973603 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(25 keys) v1 ==== 1010+0+0 (secure 0 0 0) 0x7ff4c401b9e0 con 0x7ff4d406fc10 2023-11-17T05:21:26.310 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:50.401+0000 7ff4c2ffd700 1 -- 172.21.15.40:0/3923973603 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7ff4c401a7f0 con 0x7ff4d406fc10 2023-11-17T05:21:26.310 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:50.401+0000 7ff4d89f4700 1 -- 172.21.15.40:0/3923973603 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7ff4d40fad10 con 0x7ff4d406fc10 2023-11-17T05:21:26.310 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:50.401+0000 7ff4c2ffd700 1 -- 172.21.15.40:0/3923973603 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 8) v1 ==== 36120+0+0 (secure 0 0 0) 0x7ff4c400f070 con 0x7ff4d406fc10 2023-11-17T05:21:26.310 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:50.401+0000 7ff4d89f4700 1 -- 172.21.15.40:0/3923973603 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7ff4d4045ad0 con 0x7ff4d406fc10 2023-11-17T05:21:26.310 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:50.402+0000 7ff4c2ffd700 1 --2- 172.21.15.40:0/3923973603 >> [v2:172.21.15.40:6800/929662169,v1:172.21.15.40:6801/929662169] conn(0x7ff4b802fdb0 0x7ff4b8032270 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:21:26.311 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:50.402+0000 7ff4c2ffd700 1 -- 172.21.15.40:0/3923973603 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(2..2 src has 1..2) v4 ==== 926+0+0 (secure 0 0 0) 0x7ff4c4041300 con 0x7ff4d406fc10 2023-11-17T05:21:26.311 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:50.402+0000 7ff4d1d9b700 1 --2- 172.21.15.40:0/3923973603 >> [v2:172.21.15.40:6800/929662169,v1:172.21.15.40:6801/929662169] conn(0x7ff4b802fdb0 0x7ff4b8032270 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:21:26.311 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:50.402+0000 7ff4d1d9b700 1 --2- 172.21.15.40:0/3923973603 >> [v2:172.21.15.40:6800/929662169,v1:172.21.15.40:6801/929662169] conn(0x7ff4b802fdb0 0x7ff4b8032270 secure :-1 s=READY pgs=12 cs=0 l=1 rev1=1 rx=0x7ff4c80096a0 tx=0x7ff4c8006b40).ready entity=mgr.14116 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:21:26.311 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:50.406+0000 7ff4c2ffd700 1 -- 172.21.15.40:0/3923973603 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+152212 (secure 0 0 0) 0x7ff4c40257e0 con 0x7ff4d406fc10 2023-11-17T05:21:26.311 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:50.602+0000 7ff4d89f4700 1 -- 172.21.15.40:0/3923973603 --> [v2:172.21.15.40:6800/929662169,v1:172.21.15.40:6801/929662169] -- mgr_command(tid 0: {"prefix": "orch apply", "service_type": "mgr", "target": ["mon-mgr", ""]}) v1 -- 0x7ff4d40ffd40 con 0x7ff4b802fdb0 2023-11-17T05:21:26.311 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:50.606+0000 7ff4c2ffd700 1 -- 172.21.15.40:0/3923973603 <== mgr.14116 v2:172.21.15.40:6800/929662169 1 ==== mgr_command_reply(tid 0: 0 ) v1 ==== 8+0+24 (secure 0 0 0) 0x7ff4d40ffd40 con 0x7ff4b802fdb0 2023-11-17T05:21:26.311 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: Scheduled mgr update... 2023-11-17T05:21:26.311 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:50.608+0000 7ff4d89f4700 1 -- 172.21.15.40:0/3923973603 >> [v2:172.21.15.40:6800/929662169,v1:172.21.15.40:6801/929662169] conn(0x7ff4b802fdb0 msgr2=0x7ff4b8032270 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:21:26.311 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:50.608+0000 7ff4d89f4700 1 --2- 172.21.15.40:0/3923973603 >> [v2:172.21.15.40:6800/929662169,v1:172.21.15.40:6801/929662169] conn(0x7ff4b802fdb0 0x7ff4b8032270 secure :-1 s=READY pgs=12 cs=0 l=1 rev1=1 rx=0x7ff4c80096a0 tx=0x7ff4c8006b40).stop 2023-11-17T05:21:26.311 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:50.608+0000 7ff4d89f4700 1 -- 172.21.15.40:0/3923973603 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7ff4d406fc10 msgr2=0x7ff4d40fc2e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:21:26.311 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:50.608+0000 7ff4d89f4700 1 --2- 172.21.15.40:0/3923973603 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7ff4d406fc10 0x7ff4d40fc2e0 secure :-1 s=READY pgs=44 cs=0 l=1 rev1=1 rx=0x7ff4c4011550 tx=0x7ff4c4006ee0).stop 2023-11-17T05:21:26.311 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:50.608+0000 7ff4d89f4700 1 -- 172.21.15.40:0/3923973603 shutdown_connections 2023-11-17T05:21:26.311 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:50.608+0000 7ff4d89f4700 1 --2- 172.21.15.40:0/3923973603 >> [v2:172.21.15.40:6800/929662169,v1:172.21.15.40:6801/929662169] conn(0x7ff4b802fdb0 0x7ff4b8032270 unknown :-1 s=CLOSED pgs=12 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:21:26.311 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:50.608+0000 7ff4d89f4700 1 --2- 172.21.15.40:0/3923973603 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7ff4d406fc10 0x7ff4d40fc2e0 unknown :-1 s=CLOSED pgs=44 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:21:26.312 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:50.608+0000 7ff4d89f4700 1 -- 172.21.15.40:0/3923973603 >> 172.21.15.40:0/3923973603 conn(0x7ff4d40f6280 msgr2=0x7ff4d40f7eb0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:21:26.312 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:50.608+0000 7ff4d89f4700 1 -- 172.21.15.40:0/3923973603 shutdown_connections 2023-11-17T05:21:26.312 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:50.608+0000 7ff4d89f4700 1 -- 172.21.15.40:0/3923973603 wait complete. 2023-11-17T05:21:26.312 INFO:teuthology.orchestra.run.smithi040.stderr:Deploying crash service with default placement... 2023-11-17T05:21:26.312 INFO:teuthology.orchestra.run.smithi040.stderr:Running command: /bin/podman run --rm --ipc=host --no-hosts --net=host --entrypoint /usr/bin/ceph --init -e CONTAINER_IMAGE=docker.io/ceph/ceph:v16.2.4 -e NODE_NAME=smithi040 -e CEPH_USE_RANDOM_NONCE=1 -v /var/log/ceph/ed6470e2-8508-11ee-8dbe-212e2dc638e7:/var/log/ceph:z -v /tmp/ceph-tmpzx79_7_h:/etc/ceph/ceph.client.admin.keyring:z -v /tmp/ceph-tmpru9tjirw:/etc/ceph/ceph.conf:z docker.io/ceph/ceph:v16.2.4 orch apply crash 2023-11-17T05:21:26.312 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:51.357+0000 7ff99932a700 1 Processor -- start 2023-11-17T05:21:26.312 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:51.358+0000 7ff99932a700 1 -- start start 2023-11-17T05:21:26.312 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:51.358+0000 7ff99932a700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7ff9940fb130 0x7ff9940fb550 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 rx=0 tx=0).connect 2023-11-17T05:21:26.312 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:51.358+0000 7ff99932a700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7ff9940fba40 con 0x7ff9940fb130 2023-11-17T05:21:26.312 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:51.358+0000 7ff992ffd700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7ff9940fb130 0x7ff9940fb550 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:21:26.312 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:51.358+0000 7ff992ffd700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7ff9940fb130 0x7ff9940fb550 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:38408/0 (socket says 172.21.15.40:38408) 2023-11-17T05:21:26.312 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:51.358+0000 7ff992ffd700 1 -- 172.21.15.40:0/3733323699 learned_addr learned my addr 172.21.15.40:0/3733323699 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-11-17T05:21:26.312 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:51.358+0000 7ff992ffd700 1 -- 172.21.15.40:0/3733323699 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7ff9940fbb80 con 0x7ff9940fb130 2023-11-17T05:21:26.313 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:51.358+0000 7ff992ffd700 1 --2- 172.21.15.40:0/3733323699 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7ff9940fb130 0x7ff9940fb550 secure :-1 s=READY pgs=45 cs=0 l=1 rev1=1 rx=0x7ff98400b5f0 tx=0x7ff984006ee0).ready entity=mon.0 client_cookie=bb1c51c39409bbc7 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:21:26.313 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:51.359+0000 7ff991ffb700 1 -- 172.21.15.40:0/3733323699 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7ff98400d600 con 0x7ff9940fb130 2023-11-17T05:21:26.313 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:51.359+0000 7ff991ffb700 1 -- 172.21.15.40:0/3733323699 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(25 keys) v1 ==== 1010+0+0 (secure 0 0 0) 0x7ff98400dbd0 con 0x7ff9940fb130 2023-11-17T05:21:26.313 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:51.359+0000 7ff991ffb700 1 -- 172.21.15.40:0/3733323699 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7ff984011c00 con 0x7ff9940fb130 2023-11-17T05:21:26.313 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:51.359+0000 7ff99932a700 1 -- 172.21.15.40:0/3733323699 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7ff9940fb130 msgr2=0x7ff9940fb550 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:21:26.313 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:51.359+0000 7ff99932a700 1 --2- 172.21.15.40:0/3733323699 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7ff9940fb130 0x7ff9940fb550 secure :-1 s=READY pgs=45 cs=0 l=1 rev1=1 rx=0x7ff98400b5f0 tx=0x7ff984006ee0).stop 2023-11-17T05:21:26.313 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:51.359+0000 7ff99932a700 1 -- 172.21.15.40:0/3733323699 shutdown_connections 2023-11-17T05:21:26.313 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:51.359+0000 7ff99932a700 1 --2- 172.21.15.40:0/3733323699 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7ff9940fb130 0x7ff9940fb550 unknown :-1 s=CLOSED pgs=45 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:21:26.313 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:51.359+0000 7ff99932a700 1 -- 172.21.15.40:0/3733323699 >> 172.21.15.40:0/3733323699 conn(0x7ff9940f5a80 msgr2=0x7ff9940f7ea0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:21:26.313 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:51.359+0000 7ff99932a700 1 -- 172.21.15.40:0/3733323699 shutdown_connections 2023-11-17T05:21:26.313 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:51.359+0000 7ff99932a700 1 -- 172.21.15.40:0/3733323699 wait complete. 2023-11-17T05:21:26.313 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:51.360+0000 7ff99932a700 1 Processor -- start 2023-11-17T05:21:26.313 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:51.360+0000 7ff99932a700 1 -- start start 2023-11-17T05:21:26.313 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:51.361+0000 7ff99932a700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7ff9940fb130 0x7ff9940ff490 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:21:26.314 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:51.361+0000 7ff99932a700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7ff9940fd910 con 0x7ff9940fb130 2023-11-17T05:21:26.314 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:51.361+0000 7ff992ffd700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7ff9940fb130 0x7ff9940ff490 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:21:26.314 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:51.361+0000 7ff992ffd700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7ff9940fb130 0x7ff9940ff490 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:38418/0 (socket says 172.21.15.40:38418) 2023-11-17T05:21:26.314 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:51.361+0000 7ff992ffd700 1 -- 172.21.15.40:0/899424046 learned_addr learned my addr 172.21.15.40:0/899424046 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-11-17T05:21:26.314 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:51.361+0000 7ff992ffd700 1 -- 172.21.15.40:0/899424046 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7ff98400b200 con 0x7ff9940fb130 2023-11-17T05:21:26.314 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:51.361+0000 7ff992ffd700 1 --2- 172.21.15.40:0/899424046 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7ff9940fb130 0x7ff9940ff490 secure :-1 s=READY pgs=46 cs=0 l=1 rev1=1 rx=0x7ff98400e040 tx=0x7ff984006ee0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:21:26.314 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:51.361+0000 7ff97b7fe700 1 -- 172.21.15.40:0/899424046 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7ff98401b410 con 0x7ff9940fb130 2023-11-17T05:21:26.314 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:51.361+0000 7ff97b7fe700 1 -- 172.21.15.40:0/899424046 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(25 keys) v1 ==== 1010+0+0 (secure 0 0 0) 0x7ff98401b9e0 con 0x7ff9940fb130 2023-11-17T05:21:26.314 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:51.361+0000 7ff99932a700 1 -- 172.21.15.40:0/899424046 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7ff9940fdab0 con 0x7ff9940fb130 2023-11-17T05:21:26.314 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:51.361+0000 7ff97b7fe700 1 -- 172.21.15.40:0/899424046 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7ff98401a770 con 0x7ff9940fb130 2023-11-17T05:21:26.314 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:51.362+0000 7ff99932a700 1 -- 172.21.15.40:0/899424046 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7ff9940fdec0 con 0x7ff9940fb130 2023-11-17T05:21:26.314 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:51.362+0000 7ff97b7fe700 1 -- 172.21.15.40:0/899424046 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 8) v1 ==== 36120+0+0 (secure 0 0 0) 0x7ff98400f070 con 0x7ff9940fb130 2023-11-17T05:21:26.314 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:51.363+0000 7ff97b7fe700 1 --2- 172.21.15.40:0/899424046 >> [v2:172.21.15.40:6800/929662169,v1:172.21.15.40:6801/929662169] conn(0x7ff98002fe00 0x7ff9800322c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:21:26.315 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:51.363+0000 7ff97b7fe700 1 -- 172.21.15.40:0/899424046 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(2..2 src has 1..2) v4 ==== 926+0+0 (secure 0 0 0) 0x7ff984041050 con 0x7ff9940fb130 2023-11-17T05:21:26.315 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:51.363+0000 7ff9927fc700 1 --2- 172.21.15.40:0/899424046 >> [v2:172.21.15.40:6800/929662169,v1:172.21.15.40:6801/929662169] conn(0x7ff98002fe00 0x7ff9800322c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:21:26.315 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:51.363+0000 7ff99932a700 1 -- 172.21.15.40:0/899424046 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7ff994059100 con 0x7ff9940fb130 2023-11-17T05:21:26.315 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:51.363+0000 7ff9927fc700 1 --2- 172.21.15.40:0/899424046 >> [v2:172.21.15.40:6800/929662169,v1:172.21.15.40:6801/929662169] conn(0x7ff98002fe00 0x7ff9800322c0 secure :-1 s=READY pgs=13 cs=0 l=1 rev1=1 rx=0x7ff9880096a0 tx=0x7ff988006b40).ready entity=mgr.14116 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:21:26.315 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:51.367+0000 7ff97b7fe700 1 -- 172.21.15.40:0/899424046 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+152212 (secure 0 0 0) 0x7ff984016070 con 0x7ff9940fb130 2023-11-17T05:21:26.315 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:51.563+0000 7ff99932a700 1 -- 172.21.15.40:0/899424046 --> [v2:172.21.15.40:6800/929662169,v1:172.21.15.40:6801/929662169] -- mgr_command(tid 0: {"prefix": "orch apply", "service_type": "crash", "target": ["mon-mgr", ""]}) v1 -- 0x7ff9941044e0 con 0x7ff98002fe00 2023-11-17T05:21:26.315 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:51.567+0000 7ff97b7fe700 1 -- 172.21.15.40:0/899424046 <== mgr.14116 v2:172.21.15.40:6800/929662169 1 ==== mgr_command_reply(tid 0: 0 ) v1 ==== 8+0+26 (secure 0 0 0) 0x7ff9941044e0 con 0x7ff98002fe00 2023-11-17T05:21:26.315 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: Scheduled crash update... 2023-11-17T05:21:26.315 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:51.569+0000 7ff99932a700 1 -- 172.21.15.40:0/899424046 >> [v2:172.21.15.40:6800/929662169,v1:172.21.15.40:6801/929662169] conn(0x7ff98002fe00 msgr2=0x7ff9800322c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:21:26.315 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:51.569+0000 7ff99932a700 1 --2- 172.21.15.40:0/899424046 >> [v2:172.21.15.40:6800/929662169,v1:172.21.15.40:6801/929662169] conn(0x7ff98002fe00 0x7ff9800322c0 secure :-1 s=READY pgs=13 cs=0 l=1 rev1=1 rx=0x7ff9880096a0 tx=0x7ff988006b40).stop 2023-11-17T05:21:26.315 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:51.569+0000 7ff99932a700 1 -- 172.21.15.40:0/899424046 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7ff9940fb130 msgr2=0x7ff9940ff490 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:21:26.315 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:51.569+0000 7ff99932a700 1 --2- 172.21.15.40:0/899424046 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7ff9940fb130 0x7ff9940ff490 secure :-1 s=READY pgs=46 cs=0 l=1 rev1=1 rx=0x7ff98400e040 tx=0x7ff984006ee0).stop 2023-11-17T05:21:26.315 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:51.569+0000 7ff99932a700 1 -- 172.21.15.40:0/899424046 shutdown_connections 2023-11-17T05:21:26.316 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:51.569+0000 7ff99932a700 1 --2- 172.21.15.40:0/899424046 >> [v2:172.21.15.40:6800/929662169,v1:172.21.15.40:6801/929662169] conn(0x7ff98002fe00 0x7ff9800322c0 unknown :-1 s=CLOSED pgs=13 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:21:26.316 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:51.569+0000 7ff99932a700 1 --2- 172.21.15.40:0/899424046 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7ff9940fb130 0x7ff9940ff490 unknown :-1 s=CLOSED pgs=46 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:21:26.316 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:51.569+0000 7ff99932a700 1 -- 172.21.15.40:0/899424046 >> 172.21.15.40:0/899424046 conn(0x7ff9940f5a80 msgr2=0x7ff9940fa6d0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:21:26.316 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:51.569+0000 7ff99932a700 1 -- 172.21.15.40:0/899424046 shutdown_connections 2023-11-17T05:21:26.316 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:51.569+0000 7ff99932a700 1 -- 172.21.15.40:0/899424046 wait complete. 2023-11-17T05:21:26.317 INFO:teuthology.orchestra.run.smithi040.stderr:Enabling mgr prometheus module... 2023-11-17T05:21:26.317 INFO:teuthology.orchestra.run.smithi040.stderr:Running command: /bin/podman run --rm --ipc=host --no-hosts --net=host --entrypoint /usr/bin/ceph --init -e CONTAINER_IMAGE=docker.io/ceph/ceph:v16.2.4 -e NODE_NAME=smithi040 -e CEPH_USE_RANDOM_NONCE=1 -v /var/log/ceph/ed6470e2-8508-11ee-8dbe-212e2dc638e7:/var/log/ceph:z -v /tmp/ceph-tmpzx79_7_h:/etc/ceph/ceph.client.admin.keyring:z -v /tmp/ceph-tmpru9tjirw:/etc/ceph/ceph.conf:z docker.io/ceph/ceph:v16.2.4 mgr module enable prometheus 2023-11-17T05:21:26.317 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:52.320+0000 7f6f50bd1700 1 Processor -- start 2023-11-17T05:21:26.317 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:52.320+0000 7f6f50bd1700 1 -- start start 2023-11-17T05:21:26.317 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:52.320+0000 7f6f50bd1700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f6f4c06fc90 0x7f6f4c0fe7e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 rx=0 tx=0).connect 2023-11-17T05:21:26.317 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:52.320+0000 7f6f50bd1700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f6f4c0fed60 con 0x7f6f4c06fc90 2023-11-17T05:21:26.317 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:52.321+0000 7f6f4a59c700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f6f4c06fc90 0x7f6f4c0fe7e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:21:26.317 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:52.321+0000 7f6f4a59c700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f6f4c06fc90 0x7f6f4c0fe7e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:38430/0 (socket says 172.21.15.40:38430) 2023-11-17T05:21:26.317 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:52.321+0000 7f6f4a59c700 1 -- 172.21.15.40:0/1403055114 learned_addr learned my addr 172.21.15.40:0/1403055114 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-11-17T05:21:26.317 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:52.321+0000 7f6f4a59c700 1 -- 172.21.15.40:0/1403055114 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f6f4c0feea0 con 0x7f6f4c06fc90 2023-11-17T05:21:26.317 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:52.321+0000 7f6f4a59c700 1 --2- 172.21.15.40:0/1403055114 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f6f4c06fc90 0x7f6f4c0fe7e0 secure :-1 s=READY pgs=47 cs=0 l=1 rev1=1 rx=0x7f6f3400b5f0 tx=0x7f6f34006ee0).ready entity=mon.0 client_cookie=ec486aeb5ba92f14 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:21:26.317 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:52.321+0000 7f6f4959a700 1 -- 172.21.15.40:0/1403055114 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f6f3400d600 con 0x7f6f4c06fc90 2023-11-17T05:21:26.317 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:52.321+0000 7f6f4959a700 1 -- 172.21.15.40:0/1403055114 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(25 keys) v1 ==== 1010+0+0 (secure 0 0 0) 0x7f6f3400dbd0 con 0x7f6f4c06fc90 2023-11-17T05:21:26.318 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:52.321+0000 7f6f4959a700 1 -- 172.21.15.40:0/1403055114 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f6f34011c00 con 0x7f6f4c06fc90 2023-11-17T05:21:26.318 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:52.322+0000 7f6f50bd1700 1 -- 172.21.15.40:0/1403055114 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f6f4c06fc90 msgr2=0x7f6f4c0fe7e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:21:26.318 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:52.322+0000 7f6f50bd1700 1 --2- 172.21.15.40:0/1403055114 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f6f4c06fc90 0x7f6f4c0fe7e0 secure :-1 s=READY pgs=47 cs=0 l=1 rev1=1 rx=0x7f6f3400b5f0 tx=0x7f6f34006ee0).stop 2023-11-17T05:21:26.318 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:52.322+0000 7f6f50bd1700 1 -- 172.21.15.40:0/1403055114 shutdown_connections 2023-11-17T05:21:26.318 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:52.322+0000 7f6f50bd1700 1 --2- 172.21.15.40:0/1403055114 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f6f4c06fc90 0x7f6f4c0fe7e0 unknown :-1 s=CLOSED pgs=47 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:21:26.318 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:52.322+0000 7f6f50bd1700 1 -- 172.21.15.40:0/1403055114 >> 172.21.15.40:0/1403055114 conn(0x7f6f4c0f6280 msgr2=0x7f6f4c0f86a0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:21:26.318 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:52.322+0000 7f6f50bd1700 1 -- 172.21.15.40:0/1403055114 shutdown_connections 2023-11-17T05:21:26.318 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:52.322+0000 7f6f50bd1700 1 -- 172.21.15.40:0/1403055114 wait complete. 2023-11-17T05:21:26.318 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:52.323+0000 7f6f50bd1700 1 Processor -- start 2023-11-17T05:21:26.318 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:52.323+0000 7f6f50bd1700 1 -- start start 2023-11-17T05:21:26.318 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:52.323+0000 7f6f50bd1700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f6f4c06fc90 0x7f6f4c10a9f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:21:26.318 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:52.323+0000 7f6f50bd1700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f6f4c10aee0 con 0x7f6f4c06fc90 2023-11-17T05:21:26.318 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:52.323+0000 7f6f4a59c700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f6f4c06fc90 0x7f6f4c10a9f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:21:26.319 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:52.323+0000 7f6f4a59c700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f6f4c06fc90 0x7f6f4c10a9f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:38434/0 (socket says 172.21.15.40:38434) 2023-11-17T05:21:26.319 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:52.323+0000 7f6f4a59c700 1 -- 172.21.15.40:0/4203187453 learned_addr learned my addr 172.21.15.40:0/4203187453 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-11-17T05:21:26.319 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:52.324+0000 7f6f4a59c700 1 -- 172.21.15.40:0/4203187453 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f6f3400b200 con 0x7f6f4c06fc90 2023-11-17T05:21:26.319 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:52.324+0000 7f6f4a59c700 1 --2- 172.21.15.40:0/4203187453 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f6f4c06fc90 0x7f6f4c10a9f0 secure :-1 s=READY pgs=48 cs=0 l=1 rev1=1 rx=0x7f6f34000c00 tx=0x7f6f34006ee0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:21:26.319 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:52.324+0000 7f6f42ffd700 1 -- 172.21.15.40:0/4203187453 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f6f34011690 con 0x7f6f4c06fc90 2023-11-17T05:21:26.319 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:52.324+0000 7f6f42ffd700 1 -- 172.21.15.40:0/4203187453 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(25 keys) v1 ==== 1010+0+0 (secure 0 0 0) 0x7f6f3400d9f0 con 0x7f6f4c06fc90 2023-11-17T05:21:26.319 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:52.324+0000 7f6f50bd1700 1 -- 172.21.15.40:0/4203187453 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f6f4c10b0e0 con 0x7f6f4c06fc90 2023-11-17T05:21:26.319 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:52.324+0000 7f6f42ffd700 1 -- 172.21.15.40:0/4203187453 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f6f3401b710 con 0x7f6f4c06fc90 2023-11-17T05:21:26.319 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:52.324+0000 7f6f50bd1700 1 -- 172.21.15.40:0/4203187453 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f6f4c10b4f0 con 0x7f6f4c06fc90 2023-11-17T05:21:26.319 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:52.325+0000 7f6f42ffd700 1 -- 172.21.15.40:0/4203187453 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 8) v1 ==== 36120+0+0 (secure 0 0 0) 0x7f6f3400f070 con 0x7f6f4c06fc90 2023-11-17T05:21:26.319 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:52.325+0000 7f6f42ffd700 1 --2- 172.21.15.40:0/4203187453 >> [v2:172.21.15.40:6800/929662169,v1:172.21.15.40:6801/929662169] conn(0x7f6f2c02fdb0 0x7f6f2c032270 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:21:26.319 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:52.325+0000 7f6f42ffd700 1 -- 172.21.15.40:0/4203187453 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(2..2 src has 1..2) v4 ==== 926+0+0 (secure 0 0 0) 0x7f6f340410e0 con 0x7f6f4c06fc90 2023-11-17T05:21:26.319 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:52.325+0000 7f6f49d9b700 1 --2- 172.21.15.40:0/4203187453 >> [v2:172.21.15.40:6800/929662169,v1:172.21.15.40:6801/929662169] conn(0x7f6f2c02fdb0 0x7f6f2c032270 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:21:26.319 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:52.325+0000 7f6f50bd1700 1 -- 172.21.15.40:0/4203187453 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f6f4c059100 con 0x7f6f4c06fc90 2023-11-17T05:21:26.320 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:52.325+0000 7f6f49d9b700 1 --2- 172.21.15.40:0/4203187453 >> [v2:172.21.15.40:6800/929662169,v1:172.21.15.40:6801/929662169] conn(0x7f6f2c02fdb0 0x7f6f2c032270 secure :-1 s=READY pgs=14 cs=0 l=1 rev1=1 rx=0x7f6f3c0096a0 tx=0x7f6f3c006b40).ready entity=mgr.14116 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:21:26.320 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:52.330+0000 7f6f42ffd700 1 -- 172.21.15.40:0/4203187453 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+152212 (secure 0 0 0) 0x7f6f34016070 con 0x7f6f4c06fc90 2023-11-17T05:21:26.320 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:52.536+0000 7f6f50bd1700 1 -- 172.21.15.40:0/4203187453 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "mgr module enable", "module": "prometheus"} v 0) v1 -- 0x7f6f4c045ad0 con 0x7f6f4c06fc90 2023-11-17T05:21:26.320 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:52.742+0000 7f6f42ffd700 1 -- 172.21.15.40:0/4203187453 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mgrmap(e 9) v1 ==== 36134+0+0 (secure 0 0 0) 0x7f6f3401aaf0 con 0x7f6f4c06fc90 2023-11-17T05:21:26.320 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:52.743+0000 7f6f42ffd700 1 -- 172.21.15.40:0/4203187453 <== mon.0 v2:172.21.15.40:3300/0 8 ==== mon_command_ack([{"prefix": "mgr module enable", "module": "prometheus"}]=0 v9) v1 ==== 89+0+0 (secure 0 0 0) 0x7f6f3402f8a0 con 0x7f6f4c06fc90 2023-11-17T05:21:26.320 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:52.746+0000 7f6f50bd1700 1 -- 172.21.15.40:0/4203187453 >> [v2:172.21.15.40:6800/929662169,v1:172.21.15.40:6801/929662169] conn(0x7f6f2c02fdb0 msgr2=0x7f6f2c032270 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:21:26.320 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:52.746+0000 7f6f50bd1700 1 --2- 172.21.15.40:0/4203187453 >> [v2:172.21.15.40:6800/929662169,v1:172.21.15.40:6801/929662169] conn(0x7f6f2c02fdb0 0x7f6f2c032270 secure :-1 s=READY pgs=14 cs=0 l=1 rev1=1 rx=0x7f6f3c0096a0 tx=0x7f6f3c006b40).stop 2023-11-17T05:21:26.320 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:52.746+0000 7f6f50bd1700 1 -- 172.21.15.40:0/4203187453 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f6f4c06fc90 msgr2=0x7f6f4c10a9f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:21:26.320 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:52.746+0000 7f6f50bd1700 1 --2- 172.21.15.40:0/4203187453 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f6f4c06fc90 0x7f6f4c10a9f0 secure :-1 s=READY pgs=48 cs=0 l=1 rev1=1 rx=0x7f6f34000c00 tx=0x7f6f34006ee0).stop 2023-11-17T05:21:26.320 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:52.747+0000 7f6f50bd1700 1 -- 172.21.15.40:0/4203187453 shutdown_connections 2023-11-17T05:21:26.320 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:52.747+0000 7f6f50bd1700 1 --2- 172.21.15.40:0/4203187453 >> [v2:172.21.15.40:6800/929662169,v1:172.21.15.40:6801/929662169] conn(0x7f6f2c02fdb0 0x7f6f2c032270 unknown :-1 s=CLOSED pgs=14 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:21:26.320 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:52.747+0000 7f6f50bd1700 1 --2- 172.21.15.40:0/4203187453 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f6f4c06fc90 0x7f6f4c10a9f0 unknown :-1 s=CLOSED pgs=48 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:21:26.320 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:52.747+0000 7f6f50bd1700 1 -- 172.21.15.40:0/4203187453 >> 172.21.15.40:0/4203187453 conn(0x7f6f4c0f6280 msgr2=0x7f6f4c0f7e70 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:21:26.321 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:52.747+0000 7f6f50bd1700 1 -- 172.21.15.40:0/4203187453 shutdown_connections 2023-11-17T05:21:26.321 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:52.747+0000 7f6f50bd1700 1 -- 172.21.15.40:0/4203187453 wait complete. 2023-11-17T05:21:26.321 INFO:teuthology.orchestra.run.smithi040.stderr:Deploying prometheus service with default placement... 2023-11-17T05:21:26.321 INFO:teuthology.orchestra.run.smithi040.stderr:Running command: /bin/podman run --rm --ipc=host --no-hosts --net=host --entrypoint /usr/bin/ceph --init -e CONTAINER_IMAGE=docker.io/ceph/ceph:v16.2.4 -e NODE_NAME=smithi040 -e CEPH_USE_RANDOM_NONCE=1 -v /var/log/ceph/ed6470e2-8508-11ee-8dbe-212e2dc638e7:/var/log/ceph:z -v /tmp/ceph-tmpzx79_7_h:/etc/ceph/ceph.client.admin.keyring:z -v /tmp/ceph-tmpru9tjirw:/etc/ceph/ceph.conf:z docker.io/ceph/ceph:v16.2.4 orch apply prometheus 2023-11-17T05:21:26.321 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:53.608+0000 7f3ef3219700 1 Processor -- start 2023-11-17T05:21:26.321 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:53.609+0000 7f3ef3219700 1 -- start start 2023-11-17T05:21:26.321 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:53.609+0000 7f3ef3219700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f3eec06fbb0 0x7f3eec100ad0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 rx=0 tx=0).connect 2023-11-17T05:21:26.321 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:53.609+0000 7f3ef3219700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f3eec0fc4c0 con 0x7f3eec06fbb0 2023-11-17T05:21:26.321 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:53.609+0000 7f3ef2217700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f3eec06fbb0 0x7f3eec100ad0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:21:26.321 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:53.609+0000 7f3ef2217700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f3eec06fbb0 0x7f3eec100ad0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:38464/0 (socket says 172.21.15.40:38464) 2023-11-17T05:21:26.321 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:53.609+0000 7f3ef2217700 1 -- 172.21.15.40:0/970453907 learned_addr learned my addr 172.21.15.40:0/970453907 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-11-17T05:21:26.321 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:53.610+0000 7f3ef2217700 1 -- 172.21.15.40:0/970453907 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f3eec100fc0 con 0x7f3eec06fbb0 2023-11-17T05:21:26.321 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:53.610+0000 7f3ef2217700 1 --2- 172.21.15.40:0/970453907 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f3eec06fbb0 0x7f3eec100ad0 secure :-1 s=READY pgs=51 cs=0 l=1 rev1=1 rx=0x7f3ee800b5f0 tx=0x7f3ee8006ee0).ready entity=mon.0 client_cookie=5c949c174ed68adb server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:21:26.321 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:53.610+0000 7f3ef1215700 1 -- 172.21.15.40:0/970453907 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f3ee800d600 con 0x7f3eec06fbb0 2023-11-17T05:21:26.322 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:53.610+0000 7f3ef1215700 1 -- 172.21.15.40:0/970453907 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(25 keys) v1 ==== 1010+0+0 (secure 0 0 0) 0x7f3ee800dbd0 con 0x7f3eec06fbb0 2023-11-17T05:21:26.322 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:53.610+0000 7f3ef1215700 1 -- 172.21.15.40:0/970453907 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f3ee8011c00 con 0x7f3eec06fbb0 2023-11-17T05:21:26.322 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:53.611+0000 7f3ef3219700 1 -- 172.21.15.40:0/970453907 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f3eec06fbb0 msgr2=0x7f3eec100ad0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:21:26.322 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:53.611+0000 7f3ef3219700 1 --2- 172.21.15.40:0/970453907 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f3eec06fbb0 0x7f3eec100ad0 secure :-1 s=READY pgs=51 cs=0 l=1 rev1=1 rx=0x7f3ee800b5f0 tx=0x7f3ee8006ee0).stop 2023-11-17T05:21:26.322 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:53.611+0000 7f3ef3219700 1 -- 172.21.15.40:0/970453907 shutdown_connections 2023-11-17T05:21:26.322 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:53.611+0000 7f3ef3219700 1 --2- 172.21.15.40:0/970453907 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f3eec06fbb0 0x7f3eec100ad0 unknown :-1 s=CLOSED pgs=51 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:21:26.322 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:53.611+0000 7f3ef3219700 1 -- 172.21.15.40:0/970453907 >> 172.21.15.40:0/970453907 conn(0x7f3eec0f6130 msgr2=0x7f3eec0f8550 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:21:26.322 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:53.611+0000 7f3ef3219700 1 -- 172.21.15.40:0/970453907 shutdown_connections 2023-11-17T05:21:26.322 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:53.611+0000 7f3ef3219700 1 -- 172.21.15.40:0/970453907 wait complete. 2023-11-17T05:21:26.322 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:53.612+0000 7f3ef3219700 1 Processor -- start 2023-11-17T05:21:26.322 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:53.612+0000 7f3ef3219700 1 -- start start 2023-11-17T05:21:26.322 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:53.612+0000 7f3ef3219700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f3eec06fbb0 0x7f3eec10c9d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:21:26.322 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:53.612+0000 7f3ef3219700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f3eec10cec0 con 0x7f3eec06fbb0 2023-11-17T05:21:26.323 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:53.612+0000 7f3ef2217700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f3eec06fbb0 0x7f3eec10c9d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:21:26.323 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:53.612+0000 7f3ef2217700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f3eec06fbb0 0x7f3eec10c9d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:38466/0 (socket says 172.21.15.40:38466) 2023-11-17T05:21:26.323 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:53.612+0000 7f3ef2217700 1 -- 172.21.15.40:0/3456363916 learned_addr learned my addr 172.21.15.40:0/3456363916 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-11-17T05:21:26.323 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:53.613+0000 7f3ef2217700 1 -- 172.21.15.40:0/3456363916 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f3ee800b200 con 0x7f3eec06fbb0 2023-11-17T05:21:26.323 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:53.613+0000 7f3ef2217700 1 --2- 172.21.15.40:0/3456363916 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f3eec06fbb0 0x7f3eec10c9d0 secure :-1 s=READY pgs=52 cs=0 l=1 rev1=1 rx=0x7f3ee800e040 tx=0x7f3ee8006ee0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:21:26.323 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:53.613+0000 7f3edaffd700 1 -- 172.21.15.40:0/3456363916 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f3ee800d9f0 con 0x7f3eec06fbb0 2023-11-17T05:21:26.323 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:53.613+0000 7f3edaffd700 1 -- 172.21.15.40:0/3456363916 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(25 keys) v1 ==== 1010+0+0 (secure 0 0 0) 0x7f3ee801a9f0 con 0x7f3eec06fbb0 2023-11-17T05:21:26.323 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:53.613+0000 7f3edaffd700 1 -- 172.21.15.40:0/3456363916 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f3ee801b8c0 con 0x7f3eec06fbb0 2023-11-17T05:21:26.323 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:53.614+0000 7f3ef3219700 1 -- 172.21.15.40:0/3456363916 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f3eec10d0c0 con 0x7f3eec06fbb0 2023-11-17T05:21:26.323 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:53.614+0000 7f3ef3219700 1 -- 172.21.15.40:0/3456363916 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f3eec10d4d0 con 0x7f3eec06fbb0 2023-11-17T05:21:26.323 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:53.614+0000 7f3edaffd700 1 -- 172.21.15.40:0/3456363916 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 9) v1 ==== 36134+0+0 (secure 0 0 0) 0x7f3ee800f070 con 0x7f3eec06fbb0 2023-11-17T05:21:26.323 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:53.615+0000 7f3edaffd700 1 --2- 172.21.15.40:0/3456363916 >> [v2:172.21.15.40:6800/929662169,v1:172.21.15.40:6801/929662169] conn(0x7f3ed402fe00 0x7f3ed40322c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:21:26.323 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:53.615+0000 7f3edaffd700 1 -- 172.21.15.40:0/3456363916 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(2..2 src has 1..2) v4 ==== 926+0+0 (secure 0 0 0) 0x7f3ee80413a0 con 0x7f3eec06fbb0 2023-11-17T05:21:26.323 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:53.615+0000 7f3ef1a16700 1 -- 172.21.15.40:0/3456363916 >> [v2:172.21.15.40:6800/929662169,v1:172.21.15.40:6801/929662169] conn(0x7f3ed402fe00 msgr2=0x7f3ed40322c0 unknown :-1 s=STATE_CONNECTING_RE l=1).process reconnect failed to v2:172.21.15.40:6800/929662169 2023-11-17T05:21:26.324 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:53.615+0000 7f3ef1a16700 1 --2- 172.21.15.40:0/3456363916 >> [v2:172.21.15.40:6800/929662169,v1:172.21.15.40:6801/929662169] conn(0x7f3ed402fe00 0x7f3ed40322c0 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._fault waiting 0.200000 2023-11-17T05:21:26.324 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:53.615+0000 7f3ef3219700 1 -- 172.21.15.40:0/3456363916 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f3eec059100 con 0x7f3eec06fbb0 2023-11-17T05:21:26.324 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:53.623+0000 7f3edaffd700 1 -- 172.21.15.40:0/3456363916 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+152212 (secure 0 0 0) 0x7f3ee8016070 con 0x7f3eec06fbb0 2023-11-17T05:21:26.324 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:53.816+0000 7f3ef1a16700 1 -- 172.21.15.40:0/3456363916 >> [v2:172.21.15.40:6800/929662169,v1:172.21.15.40:6801/929662169] conn(0x7f3ed402fe00 msgr2=0x7f3ed40322c0 unknown :-1 s=STATE_CONNECTING_RE l=1).process reconnect failed to v2:172.21.15.40:6800/929662169 2023-11-17T05:21:26.324 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:53.816+0000 7f3ef1a16700 1 --2- 172.21.15.40:0/3456363916 >> [v2:172.21.15.40:6800/929662169,v1:172.21.15.40:6801/929662169] conn(0x7f3ed402fe00 0x7f3ed40322c0 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._fault waiting 0.400000 2023-11-17T05:21:26.324 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:53.820+0000 7f3ef3219700 1 -- 172.21.15.40:0/3456363916 --> [v2:172.21.15.40:6800/929662169,v1:172.21.15.40:6801/929662169] -- mgr_command(tid 0: {"prefix": "orch apply", "service_type": "prometheus", "target": ["mon-mgr", ""]}) v1 -- 0x7f3eec1067f0 con 0x7f3ed402fe00 2023-11-17T05:21:26.324 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:54.217+0000 7f3ef1a16700 1 -- 172.21.15.40:0/3456363916 >> [v2:172.21.15.40:6800/929662169,v1:172.21.15.40:6801/929662169] conn(0x7f3ed402fe00 msgr2=0x7f3ed40322c0 unknown :-1 s=STATE_CONNECTING_RE l=1).process reconnect failed to v2:172.21.15.40:6800/929662169 2023-11-17T05:21:26.324 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:54.217+0000 7f3ef1a16700 1 --2- 172.21.15.40:0/3456363916 >> [v2:172.21.15.40:6800/929662169,v1:172.21.15.40:6801/929662169] conn(0x7f3ed402fe00 0x7f3ed40322c0 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._fault waiting 0.800000 2023-11-17T05:21:26.324 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:55.018+0000 7f3ef1a16700 1 -- 172.21.15.40:0/3456363916 >> [v2:172.21.15.40:6800/929662169,v1:172.21.15.40:6801/929662169] conn(0x7f3ed402fe00 msgr2=0x7f3ed40322c0 unknown :-1 s=STATE_CONNECTING_RE l=1).process reconnect failed to v2:172.21.15.40:6800/929662169 2023-11-17T05:21:26.324 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:55.018+0000 7f3ef1a16700 1 --2- 172.21.15.40:0/3456363916 >> [v2:172.21.15.40:6800/929662169,v1:172.21.15.40:6801/929662169] conn(0x7f3ed402fe00 0x7f3ed40322c0 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._fault waiting 1.600000 2023-11-17T05:21:26.324 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:56.619+0000 7f3ef1a16700 1 -- 172.21.15.40:0/3456363916 >> [v2:172.21.15.40:6800/929662169,v1:172.21.15.40:6801/929662169] conn(0x7f3ed402fe00 msgr2=0x7f3ed40322c0 unknown :-1 s=STATE_CONNECTING_RE l=1).process reconnect failed to v2:172.21.15.40:6800/929662169 2023-11-17T05:21:26.324 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:56.619+0000 7f3ef1a16700 1 --2- 172.21.15.40:0/3456363916 >> [v2:172.21.15.40:6800/929662169,v1:172.21.15.40:6801/929662169] conn(0x7f3ed402fe00 0x7f3ed40322c0 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._fault waiting 3.200000 2023-11-17T05:21:26.324 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:58.415+0000 7f3edaffd700 1 -- 172.21.15.40:0/3456363916 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mgrmap(e 10) v1 ==== 35984+0+0 (secure 0 0 0) 0x7f3ee801ab60 con 0x7f3eec06fbb0 2023-11-17T05:21:26.325 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:58.415+0000 7f3edaffd700 1 -- 172.21.15.40:0/3456363916 >> [v2:172.21.15.40:6800/929662169,v1:172.21.15.40:6801/929662169] conn(0x7f3ed402fe00 msgr2=0x7f3ed40322c0 unknown :-1 s=STATE_CONNECTING l=1).mark_down 2023-11-17T05:21:26.325 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:58.415+0000 7f3edaffd700 1 --2- 172.21.15.40:0/3456363916 >> [v2:172.21.15.40:6800/929662169,v1:172.21.15.40:6801/929662169] conn(0x7f3ed402fe00 0x7f3ed40322c0 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).stop 2023-11-17T05:21:26.325 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:59.424+0000 7f3edaffd700 1 -- 172.21.15.40:0/3456363916 <== mon.0 v2:172.21.15.40:3300/0 8 ==== mgrmap(e 11) v1 ==== 36054+0+0 (secure 0 0 0) 0x7f3ee80429c0 con 0x7f3eec06fbb0 2023-11-17T05:21:26.325 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:59.424+0000 7f3edaffd700 1 --2- 172.21.15.40:0/3456363916 >> [v2:172.21.15.40:6800/1381511815,v1:172.21.15.40:6801/1381511815] conn(0x7f3ed4033c20 0x7f3ed4036010 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:21:26.325 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:59.424+0000 7f3edaffd700 1 -- 172.21.15.40:0/3456363916 --> [v2:172.21.15.40:6800/1381511815,v1:172.21.15.40:6801/1381511815] -- mgr_command(tid 0: {"prefix": "orch apply", "service_type": "prometheus", "target": ["mon-mgr", ""]}) v1 -- 0x7f3eec1067f0 con 0x7f3ed4033c20 2023-11-17T05:21:26.325 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:59.425+0000 7f3ef1a16700 1 --2- 172.21.15.40:0/3456363916 >> [v2:172.21.15.40:6800/1381511815,v1:172.21.15.40:6801/1381511815] conn(0x7f3ed4033c20 0x7f3ed4036010 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:21:26.325 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:59.425+0000 7f3ef1a16700 1 --2- 172.21.15.40:0/3456363916 >> [v2:172.21.15.40:6800/1381511815,v1:172.21.15.40:6801/1381511815] conn(0x7f3ed4033c20 0x7f3ed4036010 secure :-1 s=READY pgs=1 cs=0 l=1 rev1=1 rx=0x7f3edc006fd0 tx=0x7f3edc006d30).ready entity=mgr.14144 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:21:26.325 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:59.431+0000 7f3edaffd700 1 -- 172.21.15.40:0/3456363916 <== mgr.14144 v2:172.21.15.40:6800/1381511815 1 ==== mgr_command_reply(tid 0: 0 ) v1 ==== 8+0+31 (secure 0 0 0) 0x7f3eec1067f0 con 0x7f3ed4033c20 2023-11-17T05:21:26.325 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: Scheduled prometheus update... 2023-11-17T05:21:26.325 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:59.434+0000 7f3ef3219700 1 -- 172.21.15.40:0/3456363916 >> [v2:172.21.15.40:6800/1381511815,v1:172.21.15.40:6801/1381511815] conn(0x7f3ed4033c20 msgr2=0x7f3ed4036010 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:21:26.325 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:59.434+0000 7f3ef3219700 1 --2- 172.21.15.40:0/3456363916 >> [v2:172.21.15.40:6800/1381511815,v1:172.21.15.40:6801/1381511815] conn(0x7f3ed4033c20 0x7f3ed4036010 secure :-1 s=READY pgs=1 cs=0 l=1 rev1=1 rx=0x7f3edc006fd0 tx=0x7f3edc006d30).stop 2023-11-17T05:21:26.325 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:59.434+0000 7f3ef3219700 1 -- 172.21.15.40:0/3456363916 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f3eec06fbb0 msgr2=0x7f3eec10c9d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:21:26.325 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:59.434+0000 7f3ef3219700 1 --2- 172.21.15.40:0/3456363916 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f3eec06fbb0 0x7f3eec10c9d0 secure :-1 s=READY pgs=52 cs=0 l=1 rev1=1 rx=0x7f3ee800e040 tx=0x7f3ee8006ee0).stop 2023-11-17T05:21:26.326 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:59.434+0000 7f3ef3219700 1 -- 172.21.15.40:0/3456363916 shutdown_connections 2023-11-17T05:21:26.326 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:59.434+0000 7f3ef3219700 1 --2- 172.21.15.40:0/3456363916 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f3eec06fbb0 0x7f3eec10c9d0 unknown :-1 s=CLOSED pgs=52 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:21:26.326 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:59.434+0000 7f3ef3219700 1 --2- 172.21.15.40:0/3456363916 >> [v2:172.21.15.40:6800/1381511815,v1:172.21.15.40:6801/1381511815] conn(0x7f3ed4033c20 0x7f3ed4036010 unknown :-1 s=CLOSED pgs=1 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:21:26.326 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:59.434+0000 7f3ef3219700 1 --2- 172.21.15.40:0/3456363916 >> [v2:172.21.15.40:6800/929662169,v1:172.21.15.40:6801/929662169] conn(0x7f3ed402fe00 0x7f3ed40322c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).stop 2023-11-17T05:21:26.326 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:59.434+0000 7f3ef3219700 1 -- 172.21.15.40:0/3456363916 >> 172.21.15.40:0/3456363916 conn(0x7f3eec0f6130 msgr2=0x7f3eec0f7c50 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:21:26.326 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:59.434+0000 7f3ef3219700 1 -- 172.21.15.40:0/3456363916 shutdown_connections 2023-11-17T05:21:26.326 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:20:59.434+0000 7f3ef3219700 1 -- 172.21.15.40:0/3456363916 wait complete. 2023-11-17T05:21:26.326 INFO:teuthology.orchestra.run.smithi040.stderr:Deploying grafana service with default placement... 2023-11-17T05:21:26.326 INFO:teuthology.orchestra.run.smithi040.stderr:Running command: /bin/podman run --rm --ipc=host --no-hosts --net=host --entrypoint /usr/bin/ceph --init -e CONTAINER_IMAGE=docker.io/ceph/ceph:v16.2.4 -e NODE_NAME=smithi040 -e CEPH_USE_RANDOM_NONCE=1 -v /var/log/ceph/ed6470e2-8508-11ee-8dbe-212e2dc638e7:/var/log/ceph:z -v /tmp/ceph-tmpzx79_7_h:/etc/ceph/ceph.client.admin.keyring:z -v /tmp/ceph-tmpru9tjirw:/etc/ceph/ceph.conf:z docker.io/ceph/ceph:v16.2.4 orch apply grafana 2023-11-17T05:21:26.326 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:00.155+0000 7fea6407a700 1 Processor -- start 2023-11-17T05:21:26.326 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:00.156+0000 7fea6407a700 1 -- start start 2023-11-17T05:21:26.326 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:00.156+0000 7fea6407a700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fea5c06fcd0 0x7fea5c0fe7a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 rx=0 tx=0).connect 2023-11-17T05:21:26.326 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:00.156+0000 7fea6407a700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fea5c0fed20 con 0x7fea5c06fcd0 2023-11-17T05:21:26.326 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:00.156+0000 7fea61e16700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fea5c06fcd0 0x7fea5c0fe7a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:21:26.327 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:00.156+0000 7fea61e16700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fea5c06fcd0 0x7fea5c0fe7a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:60790/0 (socket says 172.21.15.40:60790) 2023-11-17T05:21:26.327 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:00.156+0000 7fea61e16700 1 -- 172.21.15.40:0/2352039794 learned_addr learned my addr 172.21.15.40:0/2352039794 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-11-17T05:21:26.327 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:00.157+0000 7fea61e16700 1 -- 172.21.15.40:0/2352039794 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fea5c0fee60 con 0x7fea5c06fcd0 2023-11-17T05:21:26.327 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:00.157+0000 7fea61e16700 1 --2- 172.21.15.40:0/2352039794 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fea5c06fcd0 0x7fea5c0fe7a0 secure :-1 s=READY pgs=58 cs=0 l=1 rev1=1 rx=0x7fea5800b5f0 tx=0x7fea58006ee0).ready entity=mon.0 client_cookie=3c08ef0b6ce8bcb4 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:21:26.327 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:00.157+0000 7fea60e14700 1 -- 172.21.15.40:0/2352039794 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7fea5800d600 con 0x7fea5c06fcd0 2023-11-17T05:21:26.327 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:00.157+0000 7fea60e14700 1 -- 172.21.15.40:0/2352039794 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(25 keys) v1 ==== 1010+0+0 (secure 0 0 0) 0x7fea5800dbd0 con 0x7fea5c06fcd0 2023-11-17T05:21:26.327 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:00.157+0000 7fea60e14700 1 -- 172.21.15.40:0/2352039794 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7fea58011c00 con 0x7fea5c06fcd0 2023-11-17T05:21:26.327 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:00.158+0000 7fea6407a700 1 -- 172.21.15.40:0/2352039794 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fea5c06fcd0 msgr2=0x7fea5c0fe7a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:21:26.327 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:00.158+0000 7fea6407a700 1 --2- 172.21.15.40:0/2352039794 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fea5c06fcd0 0x7fea5c0fe7a0 secure :-1 s=READY pgs=58 cs=0 l=1 rev1=1 rx=0x7fea5800b5f0 tx=0x7fea58006ee0).stop 2023-11-17T05:21:26.327 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:00.158+0000 7fea6407a700 1 -- 172.21.15.40:0/2352039794 shutdown_connections 2023-11-17T05:21:26.327 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:00.158+0000 7fea6407a700 1 --2- 172.21.15.40:0/2352039794 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fea5c06fcd0 0x7fea5c0fe7a0 unknown :-1 s=CLOSED pgs=58 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:21:26.327 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:00.158+0000 7fea6407a700 1 -- 172.21.15.40:0/2352039794 >> 172.21.15.40:0/2352039794 conn(0x7fea5c0f6280 msgr2=0x7fea5c0f86a0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:21:26.327 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:00.158+0000 7fea6407a700 1 -- 172.21.15.40:0/2352039794 shutdown_connections 2023-11-17T05:21:26.328 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:00.158+0000 7fea6407a700 1 -- 172.21.15.40:0/2352039794 wait complete. 2023-11-17T05:21:26.328 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:00.159+0000 7fea6407a700 1 Processor -- start 2023-11-17T05:21:26.328 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:00.159+0000 7fea6407a700 1 -- start start 2023-11-17T05:21:26.328 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:00.159+0000 7fea6407a700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fea5c06fcd0 0x7fea5c10a9a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:21:26.328 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:00.159+0000 7fea6407a700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fea5c10ae90 con 0x7fea5c06fcd0 2023-11-17T05:21:26.328 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:00.159+0000 7fea61e16700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fea5c06fcd0 0x7fea5c10a9a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:21:26.328 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:00.159+0000 7fea61e16700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fea5c06fcd0 0x7fea5c10a9a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:60806/0 (socket says 172.21.15.40:60806) 2023-11-17T05:21:26.328 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:00.159+0000 7fea61e16700 1 -- 172.21.15.40:0/2205100096 learned_addr learned my addr 172.21.15.40:0/2205100096 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-11-17T05:21:26.328 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:00.159+0000 7fea61e16700 1 -- 172.21.15.40:0/2205100096 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fea5800b200 con 0x7fea5c06fcd0 2023-11-17T05:21:26.328 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:00.160+0000 7fea61e16700 1 --2- 172.21.15.40:0/2205100096 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fea5c06fcd0 0x7fea5c10a9a0 secure :-1 s=READY pgs=59 cs=0 l=1 rev1=1 rx=0x7fea5800e040 tx=0x7fea58006ee0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:21:26.328 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:00.160+0000 7fea4a7fc700 1 -- 172.21.15.40:0/2205100096 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7fea5801b410 con 0x7fea5c06fcd0 2023-11-17T05:21:26.328 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:00.160+0000 7fea4a7fc700 1 -- 172.21.15.40:0/2205100096 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(25 keys) v1 ==== 1010+0+0 (secure 0 0 0) 0x7fea5801b9e0 con 0x7fea5c06fcd0 2023-11-17T05:21:26.328 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:00.160+0000 7fea4a7fc700 1 -- 172.21.15.40:0/2205100096 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7fea5801a770 con 0x7fea5c06fcd0 2023-11-17T05:21:26.328 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:00.160+0000 7fea6407a700 1 -- 172.21.15.40:0/2205100096 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fea5c10b090 con 0x7fea5c06fcd0 2023-11-17T05:21:26.329 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:00.160+0000 7fea6407a700 1 -- 172.21.15.40:0/2205100096 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fea5c10b4a0 con 0x7fea5c06fcd0 2023-11-17T05:21:26.329 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:00.161+0000 7fea4a7fc700 1 -- 172.21.15.40:0/2205100096 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 11) v1 ==== 36054+0+0 (secure 0 0 0) 0x7fea5800f070 con 0x7fea5c06fcd0 2023-11-17T05:21:26.329 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:00.161+0000 7fea4a7fc700 1 --2- 172.21.15.40:0/2205100096 >> [v2:172.21.15.40:6800/1381511815,v1:172.21.15.40:6801/1381511815] conn(0x7fea4402fd50 0x7fea44032210 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:21:26.329 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:00.161+0000 7fea4a7fc700 1 -- 172.21.15.40:0/2205100096 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(3..3 src has 1..3) v4 ==== 1098+0+0 (secure 0 0 0) 0x7fea58040ff0 con 0x7fea5c06fcd0 2023-11-17T05:21:26.329 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:00.161+0000 7fea61615700 1 --2- 172.21.15.40:0/2205100096 >> [v2:172.21.15.40:6800/1381511815,v1:172.21.15.40:6801/1381511815] conn(0x7fea4402fd50 0x7fea44032210 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:21:26.329 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:00.161+0000 7fea6407a700 1 -- 172.21.15.40:0/2205100096 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fea5c059100 con 0x7fea5c06fcd0 2023-11-17T05:21:26.329 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:00.161+0000 7fea61615700 1 --2- 172.21.15.40:0/2205100096 >> [v2:172.21.15.40:6800/1381511815,v1:172.21.15.40:6801/1381511815] conn(0x7fea4402fd50 0x7fea44032210 secure :-1 s=READY pgs=6 cs=0 l=1 rev1=1 rx=0x7fea4c0096a0 tx=0x7fea4c006b40).ready entity=mgr.14144 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:21:26.329 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:00.166+0000 7fea4a7fc700 1 -- 172.21.15.40:0/2205100096 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+152212 (secure 0 0 0) 0x7fea58016070 con 0x7fea5c06fcd0 2023-11-17T05:21:26.329 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:00.362+0000 7fea6407a700 1 -- 172.21.15.40:0/2205100096 --> [v2:172.21.15.40:6800/1381511815,v1:172.21.15.40:6801/1381511815] -- mgr_command(tid 0: {"prefix": "orch apply", "service_type": "grafana", "target": ["mon-mgr", ""]}) v1 -- 0x7fea5c1048d0 con 0x7fea4402fd50 2023-11-17T05:21:26.329 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:00.371+0000 7fea4a7fc700 1 -- 172.21.15.40:0/2205100096 <== mgr.14144 v2:172.21.15.40:6800/1381511815 1 ==== mgr_command_reply(tid 0: 0 ) v1 ==== 8+0+28 (secure 0 0 0) 0x7fea5c1048d0 con 0x7fea4402fd50 2023-11-17T05:21:26.329 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: Scheduled grafana update... 2023-11-17T05:21:26.329 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:00.373+0000 7fea6407a700 1 -- 172.21.15.40:0/2205100096 >> [v2:172.21.15.40:6800/1381511815,v1:172.21.15.40:6801/1381511815] conn(0x7fea4402fd50 msgr2=0x7fea44032210 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:21:26.329 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:00.373+0000 7fea6407a700 1 --2- 172.21.15.40:0/2205100096 >> [v2:172.21.15.40:6800/1381511815,v1:172.21.15.40:6801/1381511815] conn(0x7fea4402fd50 0x7fea44032210 secure :-1 s=READY pgs=6 cs=0 l=1 rev1=1 rx=0x7fea4c0096a0 tx=0x7fea4c006b40).stop 2023-11-17T05:21:26.330 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:00.373+0000 7fea6407a700 1 -- 172.21.15.40:0/2205100096 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fea5c06fcd0 msgr2=0x7fea5c10a9a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:21:26.330 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:00.373+0000 7fea6407a700 1 --2- 172.21.15.40:0/2205100096 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fea5c06fcd0 0x7fea5c10a9a0 secure :-1 s=READY pgs=59 cs=0 l=1 rev1=1 rx=0x7fea5800e040 tx=0x7fea58006ee0).stop 2023-11-17T05:21:26.330 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:00.373+0000 7fea6407a700 1 -- 172.21.15.40:0/2205100096 shutdown_connections 2023-11-17T05:21:26.330 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:00.373+0000 7fea6407a700 1 --2- 172.21.15.40:0/2205100096 >> [v2:172.21.15.40:6800/1381511815,v1:172.21.15.40:6801/1381511815] conn(0x7fea4402fd50 0x7fea44032210 unknown :-1 s=CLOSED pgs=6 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:21:26.330 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:00.373+0000 7fea6407a700 1 --2- 172.21.15.40:0/2205100096 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fea5c06fcd0 0x7fea5c10a9a0 unknown :-1 s=CLOSED pgs=59 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:21:26.330 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:00.373+0000 7fea6407a700 1 -- 172.21.15.40:0/2205100096 >> 172.21.15.40:0/2205100096 conn(0x7fea5c0f6280 msgr2=0x7fea5c0f7e20 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:21:26.330 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:00.373+0000 7fea6407a700 1 -- 172.21.15.40:0/2205100096 shutdown_connections 2023-11-17T05:21:26.330 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:00.373+0000 7fea6407a700 1 -- 172.21.15.40:0/2205100096 wait complete. 2023-11-17T05:21:26.331 INFO:teuthology.orchestra.run.smithi040.stderr:Deploying node-exporter service with default placement... 2023-11-17T05:21:26.331 INFO:teuthology.orchestra.run.smithi040.stderr:Running command: /bin/podman run --rm --ipc=host --no-hosts --net=host --entrypoint /usr/bin/ceph --init -e CONTAINER_IMAGE=docker.io/ceph/ceph:v16.2.4 -e NODE_NAME=smithi040 -e CEPH_USE_RANDOM_NONCE=1 -v /var/log/ceph/ed6470e2-8508-11ee-8dbe-212e2dc638e7:/var/log/ceph:z -v /tmp/ceph-tmpzx79_7_h:/etc/ceph/ceph.client.admin.keyring:z -v /tmp/ceph-tmpru9tjirw:/etc/ceph/ceph.conf:z docker.io/ceph/ceph:v16.2.4 orch apply node-exporter 2023-11-17T05:21:26.331 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:01.136+0000 7fc47a34e700 1 Processor -- start 2023-11-17T05:21:26.331 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:01.136+0000 7fc47a34e700 1 -- start start 2023-11-17T05:21:26.331 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:01.136+0000 7fc47a34e700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fc47406fc30 0x7fc4740fe650 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 rx=0 tx=0).connect 2023-11-17T05:21:26.331 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:01.136+0000 7fc47a34e700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fc4740feb40 con 0x7fc47406fc30 2023-11-17T05:21:26.331 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:01.137+0000 7fc47934c700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fc47406fc30 0x7fc4740fe650 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:21:26.331 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:01.137+0000 7fc47934c700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fc47406fc30 0x7fc4740fe650 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:60808/0 (socket says 172.21.15.40:60808) 2023-11-17T05:21:26.331 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:01.137+0000 7fc47934c700 1 -- 172.21.15.40:0/2800431120 learned_addr learned my addr 172.21.15.40:0/2800431120 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-11-17T05:21:26.331 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:01.137+0000 7fc47934c700 1 -- 172.21.15.40:0/2800431120 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fc4740fec80 con 0x7fc47406fc30 2023-11-17T05:21:26.331 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:01.137+0000 7fc47934c700 1 --2- 172.21.15.40:0/2800431120 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fc47406fc30 0x7fc4740fe650 secure :-1 s=READY pgs=60 cs=0 l=1 rev1=1 rx=0x7fc47000b5f0 tx=0x7fc470006ee0).ready entity=mon.0 client_cookie=6bb20161382abfc0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:21:26.331 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:01.137+0000 7fc463fff700 1 -- 172.21.15.40:0/2800431120 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7fc47000d600 con 0x7fc47406fc30 2023-11-17T05:21:26.331 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:01.137+0000 7fc463fff700 1 -- 172.21.15.40:0/2800431120 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(25 keys) v1 ==== 1010+0+0 (secure 0 0 0) 0x7fc47000dbd0 con 0x7fc47406fc30 2023-11-17T05:21:26.332 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:01.137+0000 7fc463fff700 1 -- 172.21.15.40:0/2800431120 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7fc470011c00 con 0x7fc47406fc30 2023-11-17T05:21:26.332 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:01.138+0000 7fc47a34e700 1 -- 172.21.15.40:0/2800431120 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fc47406fc30 msgr2=0x7fc4740fe650 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:21:26.332 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:01.138+0000 7fc47a34e700 1 --2- 172.21.15.40:0/2800431120 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fc47406fc30 0x7fc4740fe650 secure :-1 s=READY pgs=60 cs=0 l=1 rev1=1 rx=0x7fc47000b5f0 tx=0x7fc470006ee0).stop 2023-11-17T05:21:26.332 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:01.138+0000 7fc47a34e700 1 -- 172.21.15.40:0/2800431120 shutdown_connections 2023-11-17T05:21:26.332 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:01.138+0000 7fc47a34e700 1 --2- 172.21.15.40:0/2800431120 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fc47406fc30 0x7fc4740fe650 unknown :-1 s=CLOSED pgs=60 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:21:26.332 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:01.138+0000 7fc47a34e700 1 -- 172.21.15.40:0/2800431120 >> 172.21.15.40:0/2800431120 conn(0x7fc4740f6130 msgr2=0x7fc4740f8550 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:21:26.332 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:01.138+0000 7fc47a34e700 1 -- 172.21.15.40:0/2800431120 shutdown_connections 2023-11-17T05:21:26.332 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:01.138+0000 7fc47a34e700 1 -- 172.21.15.40:0/2800431120 wait complete. 2023-11-17T05:21:26.332 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:01.139+0000 7fc47a34e700 1 Processor -- start 2023-11-17T05:21:26.332 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:01.139+0000 7fc47a34e700 1 -- start start 2023-11-17T05:21:26.332 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:01.139+0000 7fc47a34e700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fc47406fc30 0x7fc47410a760 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:21:26.332 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:01.139+0000 7fc47a34e700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fc47410ac50 con 0x7fc47406fc30 2023-11-17T05:21:26.332 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:01.139+0000 7fc47934c700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fc47406fc30 0x7fc47410a760 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:21:26.332 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:01.139+0000 7fc47934c700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fc47406fc30 0x7fc47410a760 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:60818/0 (socket says 172.21.15.40:60818) 2023-11-17T05:21:26.333 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:01.139+0000 7fc47934c700 1 -- 172.21.15.40:0/384326189 learned_addr learned my addr 172.21.15.40:0/384326189 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-11-17T05:21:26.333 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:01.139+0000 7fc47934c700 1 -- 172.21.15.40:0/384326189 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fc47000b200 con 0x7fc47406fc30 2023-11-17T05:21:26.333 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:01.139+0000 7fc47934c700 1 --2- 172.21.15.40:0/384326189 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fc47406fc30 0x7fc47410a760 secure :-1 s=READY pgs=61 cs=0 l=1 rev1=1 rx=0x7fc47000b190 tx=0x7fc470006ee0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:21:26.333 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:01.140+0000 7fc461ffb700 1 -- 172.21.15.40:0/384326189 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7fc47001b410 con 0x7fc47406fc30 2023-11-17T05:21:26.333 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:01.140+0000 7fc461ffb700 1 -- 172.21.15.40:0/384326189 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(25 keys) v1 ==== 1010+0+0 (secure 0 0 0) 0x7fc47001b9e0 con 0x7fc47406fc30 2023-11-17T05:21:26.333 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:01.140+0000 7fc47a34e700 1 -- 172.21.15.40:0/384326189 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fc47410ae50 con 0x7fc47406fc30 2023-11-17T05:21:26.333 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:01.140+0000 7fc461ffb700 1 -- 172.21.15.40:0/384326189 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7fc47001a880 con 0x7fc47406fc30 2023-11-17T05:21:26.333 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:01.140+0000 7fc47a34e700 1 -- 172.21.15.40:0/384326189 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fc47410b260 con 0x7fc47406fc30 2023-11-17T05:21:26.333 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:01.141+0000 7fc461ffb700 1 -- 172.21.15.40:0/384326189 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 11) v1 ==== 36054+0+0 (secure 0 0 0) 0x7fc47000f070 con 0x7fc47406fc30 2023-11-17T05:21:26.333 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:01.141+0000 7fc47a34e700 1 -- 172.21.15.40:0/384326189 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fc474045ad0 con 0x7fc47406fc30 2023-11-17T05:21:26.333 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:01.141+0000 7fc461ffb700 1 --2- 172.21.15.40:0/384326189 >> [v2:172.21.15.40:6800/1381511815,v1:172.21.15.40:6801/1381511815] conn(0x7fc45c02fd50 0x7fc45c032210 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:21:26.333 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:01.141+0000 7fc461ffb700 1 -- 172.21.15.40:0/384326189 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(3..3 src has 1..3) v4 ==== 1098+0+0 (secure 0 0 0) 0x7fc470041230 con 0x7fc47406fc30 2023-11-17T05:21:26.333 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:01.141+0000 7fc478b4b700 1 --2- 172.21.15.40:0/384326189 >> [v2:172.21.15.40:6800/1381511815,v1:172.21.15.40:6801/1381511815] conn(0x7fc45c02fd50 0x7fc45c032210 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:21:26.334 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:01.142+0000 7fc478b4b700 1 --2- 172.21.15.40:0/384326189 >> [v2:172.21.15.40:6800/1381511815,v1:172.21.15.40:6801/1381511815] conn(0x7fc45c02fd50 0x7fc45c032210 secure :-1 s=READY pgs=7 cs=0 l=1 rev1=1 rx=0x7fc4640096a0 tx=0x7fc464006b40).ready entity=mgr.14144 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:21:26.334 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:01.146+0000 7fc461ffb700 1 -- 172.21.15.40:0/384326189 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+152212 (secure 0 0 0) 0x7fc470019950 con 0x7fc47406fc30 2023-11-17T05:21:26.334 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:01.340+0000 7fc47a34e700 1 -- 172.21.15.40:0/384326189 --> [v2:172.21.15.40:6800/1381511815,v1:172.21.15.40:6801/1381511815] -- mgr_command(tid 0: {"prefix": "orch apply", "service_type": "node-exporter", "target": ["mon-mgr", ""]}) v1 -- 0x7fc47410bb40 con 0x7fc45c02fd50 2023-11-17T05:21:26.334 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:01.344+0000 7fc461ffb700 1 -- 172.21.15.40:0/384326189 <== mgr.14144 v2:172.21.15.40:6800/1381511815 1 ==== mgr_command_reply(tid 0: 0 ) v1 ==== 8+0+34 (secure 0 0 0) 0x7fc47410bb40 con 0x7fc45c02fd50 2023-11-17T05:21:26.334 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: Scheduled node-exporter update... 2023-11-17T05:21:26.334 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:01.346+0000 7fc47a34e700 1 -- 172.21.15.40:0/384326189 >> [v2:172.21.15.40:6800/1381511815,v1:172.21.15.40:6801/1381511815] conn(0x7fc45c02fd50 msgr2=0x7fc45c032210 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:21:26.334 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:01.346+0000 7fc47a34e700 1 --2- 172.21.15.40:0/384326189 >> [v2:172.21.15.40:6800/1381511815,v1:172.21.15.40:6801/1381511815] conn(0x7fc45c02fd50 0x7fc45c032210 secure :-1 s=READY pgs=7 cs=0 l=1 rev1=1 rx=0x7fc4640096a0 tx=0x7fc464006b40).stop 2023-11-17T05:21:26.334 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:01.346+0000 7fc47a34e700 1 -- 172.21.15.40:0/384326189 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fc47406fc30 msgr2=0x7fc47410a760 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:21:26.334 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:01.346+0000 7fc47a34e700 1 --2- 172.21.15.40:0/384326189 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fc47406fc30 0x7fc47410a760 secure :-1 s=READY pgs=61 cs=0 l=1 rev1=1 rx=0x7fc47000b190 tx=0x7fc470006ee0).stop 2023-11-17T05:21:26.334 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:01.346+0000 7fc47a34e700 1 -- 172.21.15.40:0/384326189 shutdown_connections 2023-11-17T05:21:26.334 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:01.346+0000 7fc47a34e700 1 --2- 172.21.15.40:0/384326189 >> [v2:172.21.15.40:6800/1381511815,v1:172.21.15.40:6801/1381511815] conn(0x7fc45c02fd50 0x7fc45c032210 unknown :-1 s=CLOSED pgs=7 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:21:26.334 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:01.346+0000 7fc47a34e700 1 --2- 172.21.15.40:0/384326189 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fc47406fc30 0x7fc47410a760 unknown :-1 s=CLOSED pgs=61 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:21:26.334 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:01.346+0000 7fc47a34e700 1 -- 172.21.15.40:0/384326189 >> 172.21.15.40:0/384326189 conn(0x7fc4740f6130 msgr2=0x7fc4740f7c30 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:21:26.334 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:01.346+0000 7fc47a34e700 1 -- 172.21.15.40:0/384326189 shutdown_connections 2023-11-17T05:21:26.335 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:01.346+0000 7fc47a34e700 1 -- 172.21.15.40:0/384326189 wait complete. 2023-11-17T05:21:26.335 INFO:teuthology.orchestra.run.smithi040.stderr:Deploying alertmanager service with default placement... 2023-11-17T05:21:26.335 INFO:teuthology.orchestra.run.smithi040.stderr:Running command: /bin/podman run --rm --ipc=host --no-hosts --net=host --entrypoint /usr/bin/ceph --init -e CONTAINER_IMAGE=docker.io/ceph/ceph:v16.2.4 -e NODE_NAME=smithi040 -e CEPH_USE_RANDOM_NONCE=1 -v /var/log/ceph/ed6470e2-8508-11ee-8dbe-212e2dc638e7:/var/log/ceph:z -v /tmp/ceph-tmpzx79_7_h:/etc/ceph/ceph.client.admin.keyring:z -v /tmp/ceph-tmpru9tjirw:/etc/ceph/ceph.conf:z docker.io/ceph/ceph:v16.2.4 orch apply alertmanager 2023-11-17T05:21:26.335 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:02.102+0000 7f7daa206700 1 Processor -- start 2023-11-17T05:21:26.335 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:02.102+0000 7f7daa206700 1 -- start start 2023-11-17T05:21:26.335 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:02.103+0000 7f7daa206700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f7da40fd200 0x7f7da40fd620 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 rx=0 tx=0).connect 2023-11-17T05:21:26.335 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:02.103+0000 7f7daa206700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f7da40fdb10 con 0x7f7da40fd200 2023-11-17T05:21:26.335 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:02.103+0000 7f7da37fe700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f7da40fd200 0x7f7da40fd620 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:21:26.335 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:02.103+0000 7f7da37fe700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f7da40fd200 0x7f7da40fd620 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:60826/0 (socket says 172.21.15.40:60826) 2023-11-17T05:21:26.335 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:02.103+0000 7f7da37fe700 1 -- 172.21.15.40:0/3893065452 learned_addr learned my addr 172.21.15.40:0/3893065452 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-11-17T05:21:26.335 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:02.103+0000 7f7da37fe700 1 -- 172.21.15.40:0/3893065452 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f7da40fdc50 con 0x7f7da40fd200 2023-11-17T05:21:26.335 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:02.103+0000 7f7da37fe700 1 --2- 172.21.15.40:0/3893065452 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f7da40fd200 0x7f7da40fd620 secure :-1 s=READY pgs=62 cs=0 l=1 rev1=1 rx=0x7f7d9400b5f0 tx=0x7f7d94006ee0).ready entity=mon.0 client_cookie=e6cb5b91c6a7ab6e server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:21:26.335 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:02.104+0000 7f7da27fc700 1 -- 172.21.15.40:0/3893065452 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f7d9400d600 con 0x7f7da40fd200 2023-11-17T05:21:26.336 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:02.104+0000 7f7da27fc700 1 -- 172.21.15.40:0/3893065452 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(25 keys) v1 ==== 1010+0+0 (secure 0 0 0) 0x7f7d9400dbd0 con 0x7f7da40fd200 2023-11-17T05:21:26.336 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:02.104+0000 7f7da27fc700 1 -- 172.21.15.40:0/3893065452 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f7d94011c00 con 0x7f7da40fd200 2023-11-17T05:21:26.336 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:02.104+0000 7f7daa206700 1 -- 172.21.15.40:0/3893065452 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f7da40fd200 msgr2=0x7f7da40fd620 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:21:26.336 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:02.104+0000 7f7daa206700 1 --2- 172.21.15.40:0/3893065452 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f7da40fd200 0x7f7da40fd620 secure :-1 s=READY pgs=62 cs=0 l=1 rev1=1 rx=0x7f7d9400b5f0 tx=0x7f7d94006ee0).stop 2023-11-17T05:21:26.336 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:02.104+0000 7f7daa206700 1 -- 172.21.15.40:0/3893065452 shutdown_connections 2023-11-17T05:21:26.336 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:02.104+0000 7f7daa206700 1 --2- 172.21.15.40:0/3893065452 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f7da40fd200 0x7f7da40fd620 unknown :-1 s=CLOSED pgs=62 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:21:26.336 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:02.104+0000 7f7daa206700 1 -- 172.21.15.40:0/3893065452 >> 172.21.15.40:0/3893065452 conn(0x7f7da40f5a80 msgr2=0x7f7da40f7ea0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:21:26.336 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:02.104+0000 7f7daa206700 1 -- 172.21.15.40:0/3893065452 shutdown_connections 2023-11-17T05:21:26.336 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:02.104+0000 7f7daa206700 1 -- 172.21.15.40:0/3893065452 wait complete. 2023-11-17T05:21:26.336 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:02.105+0000 7f7daa206700 1 Processor -- start 2023-11-17T05:21:26.337 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:02.105+0000 7f7daa206700 1 -- start start 2023-11-17T05:21:26.337 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:02.105+0000 7f7daa206700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f7da40fd200 0x7f7da410a9e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:21:26.337 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:02.105+0000 7f7daa206700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f7da410aed0 con 0x7f7da40fd200 2023-11-17T05:21:26.337 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:02.106+0000 7f7da37fe700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f7da40fd200 0x7f7da410a9e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:21:26.337 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:02.106+0000 7f7da37fe700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f7da40fd200 0x7f7da410a9e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:60834/0 (socket says 172.21.15.40:60834) 2023-11-17T05:21:26.337 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:02.106+0000 7f7da37fe700 1 -- 172.21.15.40:0/2008518526 learned_addr learned my addr 172.21.15.40:0/2008518526 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-11-17T05:21:26.337 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:02.106+0000 7f7da37fe700 1 -- 172.21.15.40:0/2008518526 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f7d9400b200 con 0x7f7da40fd200 2023-11-17T05:21:26.364 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:02.106+0000 7f7da37fe700 1 --2- 172.21.15.40:0/2008518526 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f7da40fd200 0x7f7da410a9e0 secure :-1 s=READY pgs=63 cs=0 l=1 rev1=1 rx=0x7f7d9400e040 tx=0x7f7d94006ee0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:21:26.364 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:02.106+0000 7f7d8bfff700 1 -- 172.21.15.40:0/2008518526 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f7d9401b410 con 0x7f7da40fd200 2023-11-17T05:21:26.365 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:02.106+0000 7f7d8bfff700 1 -- 172.21.15.40:0/2008518526 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(25 keys) v1 ==== 1010+0+0 (secure 0 0 0) 0x7f7d9401b9e0 con 0x7f7da40fd200 2023-11-17T05:21:26.365 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:02.106+0000 7f7daa206700 1 -- 172.21.15.40:0/2008518526 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f7da410b0d0 con 0x7f7da40fd200 2023-11-17T05:21:26.365 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:02.106+0000 7f7d8bfff700 1 -- 172.21.15.40:0/2008518526 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f7d9401a770 con 0x7f7da40fd200 2023-11-17T05:21:26.365 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:02.106+0000 7f7daa206700 1 -- 172.21.15.40:0/2008518526 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f7da410b4e0 con 0x7f7da40fd200 2023-11-17T05:21:26.366 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:02.107+0000 7f7d8bfff700 1 -- 172.21.15.40:0/2008518526 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 12) v1 ==== 36195+0+0 (secure 0 0 0) 0x7f7d9400f070 con 0x7f7da40fd200 2023-11-17T05:21:26.366 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:02.107+0000 7f7d8bfff700 1 --2- 172.21.15.40:0/2008518526 >> [v2:172.21.15.40:6800/1381511815,v1:172.21.15.40:6801/1381511815] conn(0x7f7d9002fe60 0x7f7d90032320 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:21:26.366 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:02.107+0000 7f7d8bfff700 1 -- 172.21.15.40:0/2008518526 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(3..3 src has 1..3) v4 ==== 1098+0+0 (secure 0 0 0) 0x7f7d940411f0 con 0x7f7da40fd200 2023-11-17T05:21:26.366 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:02.107+0000 7f7da2ffd700 1 --2- 172.21.15.40:0/2008518526 >> [v2:172.21.15.40:6800/1381511815,v1:172.21.15.40:6801/1381511815] conn(0x7f7d9002fe60 0x7f7d90032320 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:21:26.366 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:02.108+0000 7f7daa206700 1 -- 172.21.15.40:0/2008518526 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f7da4059100 con 0x7f7da40fd200 2023-11-17T05:21:26.366 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:02.108+0000 7f7da2ffd700 1 --2- 172.21.15.40:0/2008518526 >> [v2:172.21.15.40:6800/1381511815,v1:172.21.15.40:6801/1381511815] conn(0x7f7d9002fe60 0x7f7d90032320 secure :-1 s=READY pgs=8 cs=0 l=1 rev1=1 rx=0x7f7d980096a0 tx=0x7f7d98006b40).ready entity=mgr.14144 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:21:26.366 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:02.112+0000 7f7d8bfff700 1 -- 172.21.15.40:0/2008518526 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+152212 (secure 0 0 0) 0x7f7d94016070 con 0x7f7da40fd200 2023-11-17T05:21:26.366 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:02.306+0000 7f7daa206700 1 -- 172.21.15.40:0/2008518526 --> [v2:172.21.15.40:6800/1381511815,v1:172.21.15.40:6801/1381511815] -- mgr_command(tid 0: {"prefix": "orch apply", "service_type": "alertmanager", "target": ["mon-mgr", ""]}) v1 -- 0x7f7da4104550 con 0x7f7d9002fe60 2023-11-17T05:21:26.366 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:02.310+0000 7f7d8bfff700 1 -- 172.21.15.40:0/2008518526 <== mgr.14144 v2:172.21.15.40:6800/1381511815 1 ==== mgr_command_reply(tid 0: 0 ) v1 ==== 8+0+33 (secure 0 0 0) 0x7f7da4104550 con 0x7f7d9002fe60 2023-11-17T05:21:26.366 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: Scheduled alertmanager update... 2023-11-17T05:21:26.366 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:02.312+0000 7f7daa206700 1 -- 172.21.15.40:0/2008518526 >> [v2:172.21.15.40:6800/1381511815,v1:172.21.15.40:6801/1381511815] conn(0x7f7d9002fe60 msgr2=0x7f7d90032320 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:21:26.367 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:02.312+0000 7f7daa206700 1 --2- 172.21.15.40:0/2008518526 >> [v2:172.21.15.40:6800/1381511815,v1:172.21.15.40:6801/1381511815] conn(0x7f7d9002fe60 0x7f7d90032320 secure :-1 s=READY pgs=8 cs=0 l=1 rev1=1 rx=0x7f7d980096a0 tx=0x7f7d98006b40).stop 2023-11-17T05:21:26.367 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:02.312+0000 7f7daa206700 1 -- 172.21.15.40:0/2008518526 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f7da40fd200 msgr2=0x7f7da410a9e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:21:26.367 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:02.312+0000 7f7daa206700 1 --2- 172.21.15.40:0/2008518526 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f7da40fd200 0x7f7da410a9e0 secure :-1 s=READY pgs=63 cs=0 l=1 rev1=1 rx=0x7f7d9400e040 tx=0x7f7d94006ee0).stop 2023-11-17T05:21:26.367 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:02.312+0000 7f7daa206700 1 -- 172.21.15.40:0/2008518526 shutdown_connections 2023-11-17T05:21:26.367 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:02.312+0000 7f7daa206700 1 --2- 172.21.15.40:0/2008518526 >> [v2:172.21.15.40:6800/1381511815,v1:172.21.15.40:6801/1381511815] conn(0x7f7d9002fe60 0x7f7d90032320 unknown :-1 s=CLOSED pgs=8 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:21:26.367 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:02.312+0000 7f7daa206700 1 --2- 172.21.15.40:0/2008518526 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f7da40fd200 0x7f7da410a9e0 unknown :-1 s=CLOSED pgs=63 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:21:26.367 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:02.312+0000 7f7daa206700 1 -- 172.21.15.40:0/2008518526 >> 172.21.15.40:0/2008518526 conn(0x7f7da40f5a80 msgr2=0x7f7da40ff290 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:21:26.367 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:02.312+0000 7f7daa206700 1 -- 172.21.15.40:0/2008518526 shutdown_connections 2023-11-17T05:21:26.367 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:02.312+0000 7f7daa206700 1 -- 172.21.15.40:0/2008518526 wait complete. 2023-11-17T05:21:26.367 INFO:teuthology.orchestra.run.smithi040.stderr:Running command: /bin/podman run --rm --ipc=host --no-hosts --net=host --entrypoint /usr/bin/ceph --init -e CONTAINER_IMAGE=docker.io/ceph/ceph:v16.2.4 -e NODE_NAME=smithi040 -e CEPH_USE_RANDOM_NONCE=1 -v /var/log/ceph/ed6470e2-8508-11ee-8dbe-212e2dc638e7:/var/log/ceph:z -v /tmp/ceph-tmpzx79_7_h:/etc/ceph/ceph.client.admin.keyring:z -v /tmp/ceph-tmpru9tjirw:/etc/ceph/ceph.conf:z docker.io/ceph/ceph:v16.2.4 config set mgr mgr/cephadm/container_init True --force 2023-11-17T05:21:26.367 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:03.025+0000 7f190925a700 1 Processor -- start 2023-11-17T05:21:26.368 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:03.025+0000 7f190925a700 1 -- start start 2023-11-17T05:21:26.368 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:03.025+0000 7f190925a700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f190406fc70 0x7f1904100c00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 rx=0 tx=0).connect 2023-11-17T05:21:26.368 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:03.025+0000 7f190925a700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f19040fc680 con 0x7f190406fc70 2023-11-17T05:21:26.368 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:03.026+0000 7f1902d9d700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f190406fc70 0x7f1904100c00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:21:26.368 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:03.026+0000 7f1902d9d700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f190406fc70 0x7f1904100c00 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:60842/0 (socket says 172.21.15.40:60842) 2023-11-17T05:21:26.368 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:03.026+0000 7f1902d9d700 1 -- 172.21.15.40:0/3229713827 learned_addr learned my addr 172.21.15.40:0/3229713827 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-11-17T05:21:26.368 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:03.026+0000 7f1902d9d700 1 -- 172.21.15.40:0/3229713827 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f19041010f0 con 0x7f190406fc70 2023-11-17T05:21:26.368 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:03.026+0000 7f1902d9d700 1 --2- 172.21.15.40:0/3229713827 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f190406fc70 0x7f1904100c00 secure :-1 s=READY pgs=64 cs=0 l=1 rev1=1 rx=0x7f18f400b5f0 tx=0x7f18f4006ee0).ready entity=mon.0 client_cookie=1b4d81900db48148 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:21:26.368 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:03.027+0000 7f190259c700 1 -- 172.21.15.40:0/3229713827 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f18f400d600 con 0x7f190406fc70 2023-11-17T05:21:26.368 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:03.027+0000 7f190259c700 1 -- 172.21.15.40:0/3229713827 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(25 keys) v1 ==== 1010+0+0 (secure 0 0 0) 0x7f18f400dbd0 con 0x7f190406fc70 2023-11-17T05:21:26.368 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:03.027+0000 7f190259c700 1 -- 172.21.15.40:0/3229713827 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f18f4011c00 con 0x7f190406fc70 2023-11-17T05:21:26.368 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:03.027+0000 7f190925a700 1 -- 172.21.15.40:0/3229713827 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f190406fc70 msgr2=0x7f1904100c00 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:21:26.368 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:03.027+0000 7f190925a700 1 --2- 172.21.15.40:0/3229713827 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f190406fc70 0x7f1904100c00 secure :-1 s=READY pgs=64 cs=0 l=1 rev1=1 rx=0x7f18f400b5f0 tx=0x7f18f4006ee0).stop 2023-11-17T05:21:26.368 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:03.027+0000 7f190925a700 1 -- 172.21.15.40:0/3229713827 shutdown_connections 2023-11-17T05:21:26.369 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:03.027+0000 7f190925a700 1 --2- 172.21.15.40:0/3229713827 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f190406fc70 0x7f1904100c00 unknown :-1 s=CLOSED pgs=64 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:21:26.369 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:03.027+0000 7f190925a700 1 -- 172.21.15.40:0/3229713827 >> 172.21.15.40:0/3229713827 conn(0x7f19040f6280 msgr2=0x7f19040f86a0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:21:26.369 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:03.027+0000 7f190925a700 1 -- 172.21.15.40:0/3229713827 shutdown_connections 2023-11-17T05:21:26.369 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:03.027+0000 7f190925a700 1 -- 172.21.15.40:0/3229713827 wait complete. 2023-11-17T05:21:26.369 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:03.028+0000 7f190925a700 1 Processor -- start 2023-11-17T05:21:26.369 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:03.028+0000 7f190925a700 1 -- start start 2023-11-17T05:21:26.369 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:03.029+0000 7f190925a700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f190406fc70 0x7f190410cbe0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:21:26.369 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:03.029+0000 7f190925a700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f190410d0d0 con 0x7f190406fc70 2023-11-17T05:21:26.369 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:03.029+0000 7f1902d9d700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f190406fc70 0x7f190410cbe0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:21:26.369 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:03.029+0000 7f1902d9d700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f190406fc70 0x7f190410cbe0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:60844/0 (socket says 172.21.15.40:60844) 2023-11-17T05:21:26.369 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:03.029+0000 7f1902d9d700 1 -- 172.21.15.40:0/1383780309 learned_addr learned my addr 172.21.15.40:0/1383780309 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-11-17T05:21:26.369 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:03.029+0000 7f1902d9d700 1 -- 172.21.15.40:0/1383780309 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f18f400b200 con 0x7f190406fc70 2023-11-17T05:21:26.369 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:03.029+0000 7f1902d9d700 1 --2- 172.21.15.40:0/1383780309 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f190406fc70 0x7f190410cbe0 secure :-1 s=READY pgs=65 cs=0 l=1 rev1=1 rx=0x7f18f4000c00 tx=0x7f18f4006ee0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:21:26.370 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:03.029+0000 7f18fbfff700 1 -- 172.21.15.40:0/1383780309 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f18f4011690 con 0x7f190406fc70 2023-11-17T05:21:26.370 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:03.029+0000 7f18fbfff700 1 -- 172.21.15.40:0/1383780309 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(25 keys) v1 ==== 1010+0+0 (secure 0 0 0) 0x7f18f400d9f0 con 0x7f190406fc70 2023-11-17T05:21:26.370 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:03.029+0000 7f190925a700 1 -- 172.21.15.40:0/1383780309 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f190410d2d0 con 0x7f190406fc70 2023-11-17T05:21:26.370 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:03.029+0000 7f18fbfff700 1 -- 172.21.15.40:0/1383780309 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f18f401b710 con 0x7f190406fc70 2023-11-17T05:21:26.370 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:03.030+0000 7f190925a700 1 -- 172.21.15.40:0/1383780309 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f190410d6e0 con 0x7f190406fc70 2023-11-17T05:21:26.370 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:03.030+0000 7f190925a700 1 -- 172.21.15.40:0/1383780309 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f1904045ad0 con 0x7f190406fc70 2023-11-17T05:21:26.370 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:03.031+0000 7f18fbfff700 1 -- 172.21.15.40:0/1383780309 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 12) v1 ==== 36195+0+0 (secure 0 0 0) 0x7f18f400f070 con 0x7f190406fc70 2023-11-17T05:21:26.370 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:03.031+0000 7f18fbfff700 1 --2- 172.21.15.40:0/1383780309 >> [v2:172.21.15.40:6800/1381511815,v1:172.21.15.40:6801/1381511815] conn(0x7f18e402fe60 0x7f18e4032320 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:21:26.370 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:03.031+0000 7f18fbfff700 1 -- 172.21.15.40:0/1383780309 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(3..3 src has 1..3) v4 ==== 1098+0+0 (secure 0 0 0) 0x7f18f40413e0 con 0x7f190406fc70 2023-11-17T05:21:26.370 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:03.031+0000 7f18fa59c700 1 --2- 172.21.15.40:0/1383780309 >> [v2:172.21.15.40:6800/1381511815,v1:172.21.15.40:6801/1381511815] conn(0x7f18e402fe60 0x7f18e4032320 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:21:26.370 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:03.031+0000 7f18fa59c700 1 --2- 172.21.15.40:0/1383780309 >> [v2:172.21.15.40:6800/1381511815,v1:172.21.15.40:6801/1381511815] conn(0x7f18e402fe60 0x7f18e4032320 secure :-1 s=READY pgs=9 cs=0 l=1 rev1=1 rx=0x7f18ec0096a0 tx=0x7f18ec006b40).ready entity=mgr.14144 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:21:26.370 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:03.035+0000 7f18fbfff700 1 -- 172.21.15.40:0/1383780309 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+152212 (secure 0 0 0) 0x7f18f40224f0 con 0x7f190406fc70 2023-11-17T05:21:26.371 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:03.200+0000 7f190925a700 1 -- 172.21.15.40:0/1383780309 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command([{prefix=config set, name=mgr/cephadm/container_init}] v 0) v1 -- 0x7f1904101e20 con 0x7f190406fc70 2023-11-17T05:21:26.371 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:03.203+0000 7f18fbfff700 1 -- 172.21.15.40:0/1383780309 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mon_command_ack([{prefix=config set, name=mgr/cephadm/container_init}]=0 v8)=0 v8) v1 ==== 142+0+0 (secure 0 0 0) 0x7f18f4016070 con 0x7f190406fc70 2023-11-17T05:21:26.371 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:03.205+0000 7f190925a700 1 -- 172.21.15.40:0/1383780309 >> [v2:172.21.15.40:6800/1381511815,v1:172.21.15.40:6801/1381511815] conn(0x7f18e402fe60 msgr2=0x7f18e4032320 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:21:26.371 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:03.205+0000 7f190925a700 1 --2- 172.21.15.40:0/1383780309 >> [v2:172.21.15.40:6800/1381511815,v1:172.21.15.40:6801/1381511815] conn(0x7f18e402fe60 0x7f18e4032320 secure :-1 s=READY pgs=9 cs=0 l=1 rev1=1 rx=0x7f18ec0096a0 tx=0x7f18ec006b40).stop 2023-11-17T05:21:26.371 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:03.205+0000 7f190925a700 1 -- 172.21.15.40:0/1383780309 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f190406fc70 msgr2=0x7f190410cbe0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:21:26.371 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:03.205+0000 7f190925a700 1 --2- 172.21.15.40:0/1383780309 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f190406fc70 0x7f190410cbe0 secure :-1 s=READY pgs=65 cs=0 l=1 rev1=1 rx=0x7f18f4000c00 tx=0x7f18f4006ee0).stop 2023-11-17T05:21:26.371 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:03.205+0000 7f190925a700 1 -- 172.21.15.40:0/1383780309 shutdown_connections 2023-11-17T05:21:26.371 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:03.205+0000 7f190925a700 1 --2- 172.21.15.40:0/1383780309 >> [v2:172.21.15.40:6800/1381511815,v1:172.21.15.40:6801/1381511815] conn(0x7f18e402fe60 0x7f18e4032320 unknown :-1 s=CLOSED pgs=9 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:21:26.371 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:03.205+0000 7f190925a700 1 --2- 172.21.15.40:0/1383780309 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f190406fc70 0x7f190410cbe0 unknown :-1 s=CLOSED pgs=65 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:21:26.371 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:03.205+0000 7f190925a700 1 -- 172.21.15.40:0/1383780309 >> 172.21.15.40:0/1383780309 conn(0x7f19040f6280 msgr2=0x7f19040f7e80 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:21:26.371 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:03.206+0000 7f190925a700 1 -- 172.21.15.40:0/1383780309 shutdown_connections 2023-11-17T05:21:26.371 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:03.206+0000 7f190925a700 1 -- 172.21.15.40:0/1383780309 wait complete. 2023-11-17T05:21:26.372 INFO:teuthology.orchestra.run.smithi040.stderr:Running command: /bin/podman run --rm --ipc=host --no-hosts --net=host --entrypoint /usr/bin/ceph --init -e CONTAINER_IMAGE=docker.io/ceph/ceph:v16.2.4 -e NODE_NAME=smithi040 -e CEPH_USE_RANDOM_NONCE=1 -v /var/log/ceph/ed6470e2-8508-11ee-8dbe-212e2dc638e7:/var/log/ceph:z -v /tmp/ceph-tmpzx79_7_h:/etc/ceph/ceph.client.admin.keyring:z -v /tmp/ceph-tmpru9tjirw:/etc/ceph/ceph.conf:z docker.io/ceph/ceph:v16.2.4 config set mgr mgr/dashboard/ssl_server_port 8443 2023-11-17T05:21:26.372 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:03.950+0000 7f274397d700 1 Processor -- start 2023-11-17T05:21:26.372 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:03.950+0000 7f274397d700 1 -- start start 2023-11-17T05:21:26.372 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:03.951+0000 7f274397d700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f273c06fc50 0x7f273c100c20 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 rx=0 tx=0).connect 2023-11-17T05:21:26.372 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:03.951+0000 7f274397d700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f273c0fc6a0 con 0x7f273c06fc50 2023-11-17T05:21:26.372 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:03.951+0000 7f2741719700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f273c06fc50 0x7f273c100c20 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:21:26.372 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:03.951+0000 7f2741719700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f273c06fc50 0x7f273c100c20 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:60860/0 (socket says 172.21.15.40:60860) 2023-11-17T05:21:26.372 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:03.951+0000 7f2741719700 1 -- 172.21.15.40:0/601550514 learned_addr learned my addr 172.21.15.40:0/601550514 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-11-17T05:21:26.372 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:03.951+0000 7f2741719700 1 -- 172.21.15.40:0/601550514 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f273c101110 con 0x7f273c06fc50 2023-11-17T05:21:26.372 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:03.951+0000 7f2741719700 1 --2- 172.21.15.40:0/601550514 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f273c06fc50 0x7f273c100c20 secure :-1 s=READY pgs=66 cs=0 l=1 rev1=1 rx=0x7f273800b5f0 tx=0x7f2738006ee0).ready entity=mon.0 client_cookie=382f3352e4e90bec server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:21:26.372 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:03.951+0000 7f272bfff700 1 -- 172.21.15.40:0/601550514 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f273800d600 con 0x7f273c06fc50 2023-11-17T05:21:26.372 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:03.952+0000 7f272bfff700 1 -- 172.21.15.40:0/601550514 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(25 keys) v1 ==== 1010+0+0 (secure 0 0 0) 0x7f273800dbd0 con 0x7f273c06fc50 2023-11-17T05:21:26.372 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:03.952+0000 7f272bfff700 1 -- 172.21.15.40:0/601550514 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f2738011c00 con 0x7f273c06fc50 2023-11-17T05:21:26.372 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:03.952+0000 7f274397d700 1 -- 172.21.15.40:0/601550514 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f273c06fc50 msgr2=0x7f273c100c20 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:21:26.373 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:03.952+0000 7f274397d700 1 --2- 172.21.15.40:0/601550514 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f273c06fc50 0x7f273c100c20 secure :-1 s=READY pgs=66 cs=0 l=1 rev1=1 rx=0x7f273800b5f0 tx=0x7f2738006ee0).stop 2023-11-17T05:21:26.373 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:03.952+0000 7f274397d700 1 -- 172.21.15.40:0/601550514 shutdown_connections 2023-11-17T05:21:26.373 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:03.952+0000 7f274397d700 1 --2- 172.21.15.40:0/601550514 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f273c06fc50 0x7f273c100c20 unknown :-1 s=CLOSED pgs=66 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:21:26.373 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:03.952+0000 7f274397d700 1 -- 172.21.15.40:0/601550514 >> 172.21.15.40:0/601550514 conn(0x7f273c0f6280 msgr2=0x7f273c0f86a0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:21:26.373 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:03.952+0000 7f274397d700 1 -- 172.21.15.40:0/601550514 shutdown_connections 2023-11-17T05:21:26.373 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:03.952+0000 7f274397d700 1 -- 172.21.15.40:0/601550514 wait complete. 2023-11-17T05:21:26.373 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:03.953+0000 7f274397d700 1 Processor -- start 2023-11-17T05:21:26.373 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:03.953+0000 7f274397d700 1 -- start start 2023-11-17T05:21:26.373 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:03.953+0000 7f274397d700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f273c06fc50 0x7f273c10ccc0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:21:26.373 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:03.953+0000 7f274397d700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f273c10d1b0 con 0x7f273c06fc50 2023-11-17T05:21:26.373 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:03.953+0000 7f2741719700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f273c06fc50 0x7f273c10ccc0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:21:26.373 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:03.954+0000 7f2741719700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f273c06fc50 0x7f273c10ccc0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:60874/0 (socket says 172.21.15.40:60874) 2023-11-17T05:21:26.373 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:03.954+0000 7f2741719700 1 -- 172.21.15.40:0/2511549213 learned_addr learned my addr 172.21.15.40:0/2511549213 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-11-17T05:21:26.374 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:03.954+0000 7f2741719700 1 -- 172.21.15.40:0/2511549213 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f273800b200 con 0x7f273c06fc50 2023-11-17T05:21:26.374 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:03.954+0000 7f2741719700 1 --2- 172.21.15.40:0/2511549213 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f273c06fc50 0x7f273c10ccc0 secure :-1 s=READY pgs=67 cs=0 l=1 rev1=1 rx=0x7f273800b1d0 tx=0x7f2738006ee0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:21:26.374 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:03.954+0000 7f2729ffb700 1 -- 172.21.15.40:0/2511549213 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f2738011690 con 0x7f273c06fc50 2023-11-17T05:21:26.374 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:03.954+0000 7f274397d700 1 -- 172.21.15.40:0/2511549213 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f273c10d3b0 con 0x7f273c06fc50 2023-11-17T05:21:26.374 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:03.954+0000 7f2729ffb700 1 -- 172.21.15.40:0/2511549213 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(25 keys) v1 ==== 1010+0+0 (secure 0 0 0) 0x7f273801b880 con 0x7f273c06fc50 2023-11-17T05:21:26.374 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:03.954+0000 7f2729ffb700 1 -- 172.21.15.40:0/2511549213 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f273801a650 con 0x7f273c06fc50 2023-11-17T05:21:26.374 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:03.955+0000 7f274397d700 1 -- 172.21.15.40:0/2511549213 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f273c10d7c0 con 0x7f273c06fc50 2023-11-17T05:21:26.374 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:03.955+0000 7f2729ffb700 1 -- 172.21.15.40:0/2511549213 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 12) v1 ==== 36195+0+0 (secure 0 0 0) 0x7f273800f070 con 0x7f273c06fc50 2023-11-17T05:21:26.374 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:03.956+0000 7f2729ffb700 1 --2- 172.21.15.40:0/2511549213 >> [v2:172.21.15.40:6800/1381511815,v1:172.21.15.40:6801/1381511815] conn(0x7f272402fe60 0x7f2724032320 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:21:26.374 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:03.956+0000 7f274397d700 1 -- 172.21.15.40:0/2511549213 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f273c045ad0 con 0x7f273c06fc50 2023-11-17T05:21:26.374 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:03.956+0000 7f2729ffb700 1 -- 172.21.15.40:0/2511549213 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(3..3 src has 1..3) v4 ==== 1098+0+0 (secure 0 0 0) 0x7f27380412b0 con 0x7f273c06fc50 2023-11-17T05:21:26.374 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:03.956+0000 7f2740f18700 1 --2- 172.21.15.40:0/2511549213 >> [v2:172.21.15.40:6800/1381511815,v1:172.21.15.40:6801/1381511815] conn(0x7f272402fe60 0x7f2724032320 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:21:26.374 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:03.956+0000 7f2740f18700 1 --2- 172.21.15.40:0/2511549213 >> [v2:172.21.15.40:6800/1381511815,v1:172.21.15.40:6801/1381511815] conn(0x7f272402fe60 0x7f2724032320 secure :-1 s=READY pgs=10 cs=0 l=1 rev1=1 rx=0x7f272c0096a0 tx=0x7f272c006b40).ready entity=mgr.14144 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:21:26.374 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:03.960+0000 7f2729ffb700 1 -- 172.21.15.40:0/2511549213 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+152212 (secure 0 0 0) 0x7f27380224f0 con 0x7f273c06fc50 2023-11-17T05:21:26.375 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:04.125+0000 7f274397d700 1 -- 172.21.15.40:0/2511549213 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command([{prefix=config set, name=mgr/dashboard/ssl_server_port}] v 0) v1 -- 0x7f273c101de0 con 0x7f273c06fc50 2023-11-17T05:21:26.375 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:04.129+0000 7f2729ffb700 1 -- 172.21.15.40:0/2511549213 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mon_command_ack([{prefix=config set, name=mgr/dashboard/ssl_server_port}]=0 v9)=0 v9) v1 ==== 130+0+0 (secure 0 0 0) 0x7f2738016030 con 0x7f273c06fc50 2023-11-17T05:21:26.375 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:04.131+0000 7f274397d700 1 -- 172.21.15.40:0/2511549213 >> [v2:172.21.15.40:6800/1381511815,v1:172.21.15.40:6801/1381511815] conn(0x7f272402fe60 msgr2=0x7f2724032320 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:21:26.375 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:04.131+0000 7f274397d700 1 --2- 172.21.15.40:0/2511549213 >> [v2:172.21.15.40:6800/1381511815,v1:172.21.15.40:6801/1381511815] conn(0x7f272402fe60 0x7f2724032320 secure :-1 s=READY pgs=10 cs=0 l=1 rev1=1 rx=0x7f272c0096a0 tx=0x7f272c006b40).stop 2023-11-17T05:21:26.375 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:04.131+0000 7f274397d700 1 -- 172.21.15.40:0/2511549213 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f273c06fc50 msgr2=0x7f273c10ccc0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:21:26.375 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:04.131+0000 7f274397d700 1 --2- 172.21.15.40:0/2511549213 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f273c06fc50 0x7f273c10ccc0 secure :-1 s=READY pgs=67 cs=0 l=1 rev1=1 rx=0x7f273800b1d0 tx=0x7f2738006ee0).stop 2023-11-17T05:21:26.375 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:04.131+0000 7f274397d700 1 -- 172.21.15.40:0/2511549213 shutdown_connections 2023-11-17T05:21:26.375 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:04.131+0000 7f274397d700 1 --2- 172.21.15.40:0/2511549213 >> [v2:172.21.15.40:6800/1381511815,v1:172.21.15.40:6801/1381511815] conn(0x7f272402fe60 0x7f2724032320 unknown :-1 s=CLOSED pgs=10 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:21:26.375 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:04.131+0000 7f274397d700 1 --2- 172.21.15.40:0/2511549213 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f273c06fc50 0x7f273c10ccc0 unknown :-1 s=CLOSED pgs=67 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:21:26.375 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:04.131+0000 7f274397d700 1 -- 172.21.15.40:0/2511549213 >> 172.21.15.40:0/2511549213 conn(0x7f273c0f6280 msgr2=0x7f273c0f7f10 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:21:26.375 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:04.131+0000 7f274397d700 1 -- 172.21.15.40:0/2511549213 shutdown_connections 2023-11-17T05:21:26.375 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:04.131+0000 7f274397d700 1 -- 172.21.15.40:0/2511549213 wait complete. 2023-11-17T05:21:26.375 INFO:teuthology.orchestra.run.smithi040.stderr:Enabling the dashboard module... 2023-11-17T05:21:26.376 INFO:teuthology.orchestra.run.smithi040.stderr:Running command: /bin/podman run --rm --ipc=host --no-hosts --net=host --entrypoint /usr/bin/ceph --init -e CONTAINER_IMAGE=docker.io/ceph/ceph:v16.2.4 -e NODE_NAME=smithi040 -e CEPH_USE_RANDOM_NONCE=1 -v /var/log/ceph/ed6470e2-8508-11ee-8dbe-212e2dc638e7:/var/log/ceph:z -v /tmp/ceph-tmpzx79_7_h:/etc/ceph/ceph.client.admin.keyring:z -v /tmp/ceph-tmpru9tjirw:/etc/ceph/ceph.conf:z docker.io/ceph/ceph:v16.2.4 mgr module enable dashboard 2023-11-17T05:21:26.376 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:04.865+0000 7f0db4d37700 1 Processor -- start 2023-11-17T05:21:26.376 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:04.865+0000 7f0db4d37700 1 -- start start 2023-11-17T05:21:26.377 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:04.866+0000 7f0db4d37700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f0db00fd200 0x7f0db00fd620 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 rx=0 tx=0).connect 2023-11-17T05:21:26.377 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:04.866+0000 7f0db4d37700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f0db00fdb10 con 0x7f0db00fd200 2023-11-17T05:21:26.377 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:04.866+0000 7f0dae59c700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f0db00fd200 0x7f0db00fd620 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:21:26.377 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:04.866+0000 7f0dae59c700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f0db00fd200 0x7f0db00fd620 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:45606/0 (socket says 172.21.15.40:45606) 2023-11-17T05:21:26.377 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:04.866+0000 7f0dae59c700 1 -- 172.21.15.40:0/3080714779 learned_addr learned my addr 172.21.15.40:0/3080714779 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-11-17T05:21:26.377 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:04.866+0000 7f0dae59c700 1 -- 172.21.15.40:0/3080714779 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f0db00fdc50 con 0x7f0db00fd200 2023-11-17T05:21:26.377 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:04.866+0000 7f0dae59c700 1 --2- 172.21.15.40:0/3080714779 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f0db00fd200 0x7f0db00fd620 secure :-1 s=READY pgs=68 cs=0 l=1 rev1=1 rx=0x7f0da000b5f0 tx=0x7f0da0006ee0).ready entity=mon.0 client_cookie=e52282fd6505114d server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:21:26.377 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:04.867+0000 7f0dad59a700 1 -- 172.21.15.40:0/3080714779 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f0da000d600 con 0x7f0db00fd200 2023-11-17T05:21:26.378 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:04.867+0000 7f0dad59a700 1 -- 172.21.15.40:0/3080714779 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(25 keys) v1 ==== 1010+0+0 (secure 0 0 0) 0x7f0da000dbd0 con 0x7f0db00fd200 2023-11-17T05:21:26.378 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:04.867+0000 7f0dad59a700 1 -- 172.21.15.40:0/3080714779 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f0da0011c00 con 0x7f0db00fd200 2023-11-17T05:21:26.378 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:04.867+0000 7f0db4d37700 1 -- 172.21.15.40:0/3080714779 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f0db00fd200 msgr2=0x7f0db00fd620 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:21:26.379 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:04.867+0000 7f0db4d37700 1 --2- 172.21.15.40:0/3080714779 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f0db00fd200 0x7f0db00fd620 secure :-1 s=READY pgs=68 cs=0 l=1 rev1=1 rx=0x7f0da000b5f0 tx=0x7f0da0006ee0).stop 2023-11-17T05:21:26.379 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:04.867+0000 7f0db4d37700 1 -- 172.21.15.40:0/3080714779 shutdown_connections 2023-11-17T05:21:26.379 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:04.867+0000 7f0db4d37700 1 --2- 172.21.15.40:0/3080714779 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f0db00fd200 0x7f0db00fd620 unknown :-1 s=CLOSED pgs=68 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:21:26.380 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:04.867+0000 7f0db4d37700 1 -- 172.21.15.40:0/3080714779 >> 172.21.15.40:0/3080714779 conn(0x7f0db00f5a80 msgr2=0x7f0db00f7ea0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:21:26.380 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:04.867+0000 7f0db4d37700 1 -- 172.21.15.40:0/3080714779 shutdown_connections 2023-11-17T05:21:26.380 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:04.867+0000 7f0db4d37700 1 -- 172.21.15.40:0/3080714779 wait complete. 2023-11-17T05:21:26.380 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:04.868+0000 7f0db4d37700 1 Processor -- start 2023-11-17T05:21:26.380 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:04.868+0000 7f0db4d37700 1 -- start start 2023-11-17T05:21:26.380 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:04.868+0000 7f0db4d37700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f0db00fd200 0x7f0db00fcad0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:21:26.380 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:04.868+0000 7f0db4d37700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f0db00faf50 con 0x7f0db00fd200 2023-11-17T05:21:26.380 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:04.868+0000 7f0dae59c700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f0db00fd200 0x7f0db00fcad0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:21:26.385 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:04.868+0000 7f0dae59c700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f0db00fd200 0x7f0db00fcad0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:45608/0 (socket says 172.21.15.40:45608) 2023-11-17T05:21:26.385 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:04.868+0000 7f0dae59c700 1 -- 172.21.15.40:0/4006425044 learned_addr learned my addr 172.21.15.40:0/4006425044 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-11-17T05:21:26.385 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:04.869+0000 7f0dae59c700 1 -- 172.21.15.40:0/4006425044 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f0da000b200 con 0x7f0db00fd200 2023-11-17T05:21:26.385 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:04.869+0000 7f0dae59c700 1 --2- 172.21.15.40:0/4006425044 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f0db00fd200 0x7f0db00fcad0 secure :-1 s=READY pgs=69 cs=0 l=1 rev1=1 rx=0x7f0da000e040 tx=0x7f0da0006ee0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:21:26.386 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:04.869+0000 7f0d9effd700 1 -- 172.21.15.40:0/4006425044 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f0da001b410 con 0x7f0db00fd200 2023-11-17T05:21:26.386 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:04.869+0000 7f0d9effd700 1 -- 172.21.15.40:0/4006425044 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(25 keys) v1 ==== 1010+0+0 (secure 0 0 0) 0x7f0da001b9e0 con 0x7f0db00fd200 2023-11-17T05:21:26.386 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:04.869+0000 7f0db4d37700 1 -- 172.21.15.40:0/4006425044 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f0db00fb0f0 con 0x7f0db00fd200 2023-11-17T05:21:26.386 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:04.869+0000 7f0d9effd700 1 -- 172.21.15.40:0/4006425044 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f0da001a770 con 0x7f0db00fd200 2023-11-17T05:21:26.386 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:04.869+0000 7f0db4d37700 1 -- 172.21.15.40:0/4006425044 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f0db00fb500 con 0x7f0db00fd200 2023-11-17T05:21:26.386 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:04.870+0000 7f0d9effd700 1 -- 172.21.15.40:0/4006425044 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 12) v1 ==== 36195+0+0 (secure 0 0 0) 0x7f0da000f070 con 0x7f0db00fd200 2023-11-17T05:21:26.386 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:04.870+0000 7f0db4d37700 1 -- 172.21.15.40:0/4006425044 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f0db0045ad0 con 0x7f0db00fd200 2023-11-17T05:21:26.386 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:04.870+0000 7f0d9effd700 1 --2- 172.21.15.40:0/4006425044 >> [v2:172.21.15.40:6800/1381511815,v1:172.21.15.40:6801/1381511815] conn(0x7f0d9402fe60 0x7f0d94032320 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:21:26.386 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:04.870+0000 7f0d9effd700 1 -- 172.21.15.40:0/4006425044 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(3..3 src has 1..3) v4 ==== 1098+0+0 (secure 0 0 0) 0x7f0da00411b0 con 0x7f0db00fd200 2023-11-17T05:21:26.386 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:04.871+0000 7f0dadd9b700 1 --2- 172.21.15.40:0/4006425044 >> [v2:172.21.15.40:6800/1381511815,v1:172.21.15.40:6801/1381511815] conn(0x7f0d9402fe60 0x7f0d94032320 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:21:26.387 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:04.871+0000 7f0dadd9b700 1 --2- 172.21.15.40:0/4006425044 >> [v2:172.21.15.40:6800/1381511815,v1:172.21.15.40:6801/1381511815] conn(0x7f0d9402fe60 0x7f0d94032320 secure :-1 s=READY pgs=11 cs=0 l=1 rev1=1 rx=0x7f0da40096a0 tx=0x7f0da4006b40).ready entity=mgr.14144 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:21:26.387 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:04.875+0000 7f0d9effd700 1 -- 172.21.15.40:0/4006425044 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+152212 (secure 0 0 0) 0x7f0da001a9e0 con 0x7f0db00fd200 2023-11-17T05:21:26.387 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:05.083+0000 7f0db4d37700 1 -- 172.21.15.40:0/4006425044 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "mgr module enable", "module": "dashboard"} v 0) v1 -- 0x7f0db00fa620 con 0x7f0db00fd200 2023-11-17T05:21:26.387 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:05.393+0000 7f0d9effd700 1 -- 172.21.15.40:0/4006425044 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mgrmap(e 13) v1 ==== 36208+0+0 (secure 0 0 0) 0x7f0da0019d70 con 0x7f0db00fd200 2023-11-17T05:21:26.387 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:05.395+0000 7f0d9effd700 1 -- 172.21.15.40:0/4006425044 <== mon.0 v2:172.21.15.40:3300/0 8 ==== mon_command_ack([{"prefix": "mgr module enable", "module": "dashboard"}]=0 v13) v1 ==== 88+0+0 (secure 0 0 0) 0x7f0da0022310 con 0x7f0db00fd200 2023-11-17T05:21:26.387 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:05.398+0000 7f0db4d37700 1 -- 172.21.15.40:0/4006425044 >> [v2:172.21.15.40:6800/1381511815,v1:172.21.15.40:6801/1381511815] conn(0x7f0d9402fe60 msgr2=0x7f0d94032320 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:21:26.388 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:05.398+0000 7f0db4d37700 1 --2- 172.21.15.40:0/4006425044 >> [v2:172.21.15.40:6800/1381511815,v1:172.21.15.40:6801/1381511815] conn(0x7f0d9402fe60 0x7f0d94032320 secure :-1 s=READY pgs=11 cs=0 l=1 rev1=1 rx=0x7f0da40096a0 tx=0x7f0da4006b40).stop 2023-11-17T05:21:26.388 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:05.398+0000 7f0db4d37700 1 -- 172.21.15.40:0/4006425044 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f0db00fd200 msgr2=0x7f0db00fcad0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:21:26.388 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:05.398+0000 7f0db4d37700 1 --2- 172.21.15.40:0/4006425044 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f0db00fd200 0x7f0db00fcad0 secure :-1 s=READY pgs=69 cs=0 l=1 rev1=1 rx=0x7f0da000e040 tx=0x7f0da0006ee0).stop 2023-11-17T05:21:26.388 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:05.398+0000 7f0db4d37700 1 -- 172.21.15.40:0/4006425044 shutdown_connections 2023-11-17T05:21:26.388 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:05.398+0000 7f0db4d37700 1 --2- 172.21.15.40:0/4006425044 >> [v2:172.21.15.40:6800/1381511815,v1:172.21.15.40:6801/1381511815] conn(0x7f0d9402fe60 0x7f0d94032320 unknown :-1 s=CLOSED pgs=11 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:21:26.388 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:05.398+0000 7f0db4d37700 1 --2- 172.21.15.40:0/4006425044 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f0db00fd200 0x7f0db00fcad0 unknown :-1 s=CLOSED pgs=69 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:21:26.388 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:05.398+0000 7f0db4d37700 1 -- 172.21.15.40:0/4006425044 >> 172.21.15.40:0/4006425044 conn(0x7f0db00f5a80 msgr2=0x7f0db00ff2c0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:21:26.388 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:05.398+0000 7f0db4d37700 1 -- 172.21.15.40:0/4006425044 shutdown_connections 2023-11-17T05:21:26.388 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:05.398+0000 7f0db4d37700 1 -- 172.21.15.40:0/4006425044 wait complete. 2023-11-17T05:21:26.388 INFO:teuthology.orchestra.run.smithi040.stderr:Running command: /bin/podman run --rm --ipc=host --no-hosts --net=host --entrypoint /usr/bin/ceph --init -e CONTAINER_IMAGE=docker.io/ceph/ceph:v16.2.4 -e NODE_NAME=smithi040 -e CEPH_USE_RANDOM_NONCE=1 -v /var/log/ceph/ed6470e2-8508-11ee-8dbe-212e2dc638e7:/var/log/ceph:z -v /tmp/ceph-tmpzx79_7_h:/etc/ceph/ceph.client.admin.keyring:z -v /tmp/ceph-tmpru9tjirw:/etc/ceph/ceph.conf:z docker.io/ceph/ceph:v16.2.4 mgr stat 2023-11-17T05:21:26.388 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:06.311+0000 7feae789e700 1 Processor -- start 2023-11-17T05:21:26.388 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:06.312+0000 7feae789e700 1 -- start start 2023-11-17T05:21:26.389 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:06.312+0000 7feae789e700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7feae006fc90 0x7feae00fe7e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 rx=0 tx=0).connect 2023-11-17T05:21:26.389 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:06.312+0000 7feae789e700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7feae00fed60 con 0x7feae006fc90 2023-11-17T05:21:26.389 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:06.312+0000 7feae563a700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7feae006fc90 0x7feae00fe7e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:21:26.389 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:06.312+0000 7feae563a700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7feae006fc90 0x7feae00fe7e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:45638/0 (socket says 172.21.15.40:45638) 2023-11-17T05:21:26.389 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:06.312+0000 7feae563a700 1 -- 172.21.15.40:0/182351114 learned_addr learned my addr 172.21.15.40:0/182351114 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-11-17T05:21:26.389 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:06.312+0000 7feae563a700 1 -- 172.21.15.40:0/182351114 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7feae00feea0 con 0x7feae006fc90 2023-11-17T05:21:26.389 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:06.313+0000 7feae563a700 1 --2- 172.21.15.40:0/182351114 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7feae006fc90 0x7feae00fe7e0 secure :-1 s=READY pgs=72 cs=0 l=1 rev1=1 rx=0x7feadc00b5f0 tx=0x7feadc006ee0).ready entity=mon.0 client_cookie=b35fc4ee5b77bda0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:21:26.389 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:06.313+0000 7feacffff700 1 -- 172.21.15.40:0/182351114 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7feadc00d600 con 0x7feae006fc90 2023-11-17T05:21:26.389 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:06.313+0000 7feacffff700 1 -- 172.21.15.40:0/182351114 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(25 keys) v1 ==== 1010+0+0 (secure 0 0 0) 0x7feadc00dbd0 con 0x7feae006fc90 2023-11-17T05:21:26.390 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:06.313+0000 7feacffff700 1 -- 172.21.15.40:0/182351114 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7feadc011c00 con 0x7feae006fc90 2023-11-17T05:21:26.390 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:06.314+0000 7feae789e700 1 -- 172.21.15.40:0/182351114 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7feae006fc90 msgr2=0x7feae00fe7e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:21:26.390 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:06.314+0000 7feae789e700 1 --2- 172.21.15.40:0/182351114 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7feae006fc90 0x7feae00fe7e0 secure :-1 s=READY pgs=72 cs=0 l=1 rev1=1 rx=0x7feadc00b5f0 tx=0x7feadc006ee0).stop 2023-11-17T05:21:26.390 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:06.314+0000 7feae789e700 1 -- 172.21.15.40:0/182351114 shutdown_connections 2023-11-17T05:21:26.390 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:06.314+0000 7feae789e700 1 --2- 172.21.15.40:0/182351114 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7feae006fc90 0x7feae00fe7e0 unknown :-1 s=CLOSED pgs=72 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:21:26.390 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:06.314+0000 7feae789e700 1 -- 172.21.15.40:0/182351114 >> 172.21.15.40:0/182351114 conn(0x7feae00f6280 msgr2=0x7feae00f86a0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:21:26.390 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:06.314+0000 7feae789e700 1 -- 172.21.15.40:0/182351114 shutdown_connections 2023-11-17T05:21:26.390 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:06.314+0000 7feae789e700 1 -- 172.21.15.40:0/182351114 wait complete. 2023-11-17T05:21:26.390 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:06.315+0000 7feae789e700 1 Processor -- start 2023-11-17T05:21:26.390 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:06.315+0000 7feae789e700 1 -- start start 2023-11-17T05:21:26.390 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:06.315+0000 7feae789e700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7feae006fc90 0x7feae010a9f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:21:26.390 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:06.315+0000 7feae789e700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7feae010aee0 con 0x7feae006fc90 2023-11-17T05:21:26.391 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:06.315+0000 7feae563a700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7feae006fc90 0x7feae010a9f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:21:26.391 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:06.315+0000 7feae563a700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7feae006fc90 0x7feae010a9f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:45648/0 (socket says 172.21.15.40:45648) 2023-11-17T05:21:26.391 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:06.315+0000 7feae563a700 1 -- 172.21.15.40:0/3630407038 learned_addr learned my addr 172.21.15.40:0/3630407038 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-11-17T05:21:26.391 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:06.316+0000 7feae563a700 1 -- 172.21.15.40:0/3630407038 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7feadc00b200 con 0x7feae006fc90 2023-11-17T05:21:26.391 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:06.316+0000 7feae563a700 1 --2- 172.21.15.40:0/3630407038 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7feae006fc90 0x7feae010a9f0 secure :-1 s=READY pgs=73 cs=0 l=1 rev1=1 rx=0x7feadc000c00 tx=0x7feadc006ee0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:21:26.391 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:06.316+0000 7feacdffb700 1 -- 172.21.15.40:0/3630407038 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7feadc011690 con 0x7feae006fc90 2023-11-17T05:21:26.391 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:06.316+0000 7feacdffb700 1 -- 172.21.15.40:0/3630407038 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(25 keys) v1 ==== 1010+0+0 (secure 0 0 0) 0x7feadc00d9f0 con 0x7feae006fc90 2023-11-17T05:21:26.391 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:06.316+0000 7feacdffb700 1 -- 172.21.15.40:0/3630407038 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7feadc01b710 con 0x7feae006fc90 2023-11-17T05:21:26.391 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:06.316+0000 7feae789e700 1 -- 172.21.15.40:0/3630407038 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7feae010b0e0 con 0x7feae006fc90 2023-11-17T05:21:26.391 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:06.316+0000 7feae789e700 1 -- 172.21.15.40:0/3630407038 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7feae010b4f0 con 0x7feae006fc90 2023-11-17T05:21:26.391 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:06.317+0000 7feacdffb700 1 -- 172.21.15.40:0/3630407038 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 13) v1 ==== 36208+0+0 (secure 0 0 0) 0x7feadc00f070 con 0x7feae006fc90 2023-11-17T05:21:26.391 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:06.317+0000 7feacdffb700 1 --2- 172.21.15.40:0/3630407038 >> [v2:172.21.15.40:6800/1381511815,v1:172.21.15.40:6801/1381511815] conn(0x7feac802ff00 0x7feac80323c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:21:26.391 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:06.317+0000 7feacdffb700 1 -- 172.21.15.40:0/3630407038 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(3..3 src has 1..3) v4 ==== 1098+0+0 (secure 0 0 0) 0x7feadc041440 con 0x7feae006fc90 2023-11-17T05:21:26.392 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:06.317+0000 7feae4e39700 1 -- 172.21.15.40:0/3630407038 >> [v2:172.21.15.40:6800/1381511815,v1:172.21.15.40:6801/1381511815] conn(0x7feac802ff00 msgr2=0x7feac80323c0 unknown :-1 s=STATE_CONNECTING_RE l=1).process reconnect failed to v2:172.21.15.40:6800/1381511815 2023-11-17T05:21:26.392 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:06.318+0000 7feae4e39700 1 --2- 172.21.15.40:0/3630407038 >> [v2:172.21.15.40:6800/1381511815,v1:172.21.15.40:6801/1381511815] conn(0x7feac802ff00 0x7feac80323c0 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._fault waiting 0.200000 2023-11-17T05:21:26.393 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:06.318+0000 7feae789e700 1 -- 172.21.15.40:0/3630407038 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7feae0059100 con 0x7feae006fc90 2023-11-17T05:21:26.393 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:06.322+0000 7feacdffb700 1 -- 172.21.15.40:0/3630407038 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+152212 (secure 0 0 0) 0x7feadc016070 con 0x7feae006fc90 2023-11-17T05:21:26.393 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:06.519+0000 7feae4e39700 1 -- 172.21.15.40:0/3630407038 >> [v2:172.21.15.40:6800/1381511815,v1:172.21.15.40:6801/1381511815] conn(0x7feac802ff00 msgr2=0x7feac80323c0 unknown :-1 s=STATE_CONNECTING_RE l=1).process reconnect failed to v2:172.21.15.40:6800/1381511815 2023-11-17T05:21:26.393 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:06.519+0000 7feae4e39700 1 --2- 172.21.15.40:0/3630407038 >> [v2:172.21.15.40:6800/1381511815,v1:172.21.15.40:6801/1381511815] conn(0x7feac802ff00 0x7feac80323c0 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._fault waiting 0.400000 2023-11-17T05:21:26.393 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:06.532+0000 7feae789e700 1 -- 172.21.15.40:0/3630407038 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "mgr stat"} v 0) v1 -- 0x7feae00ffbd0 con 0x7feae006fc90 2023-11-17T05:21:26.393 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:06.533+0000 7feacdffb700 1 -- 172.21.15.40:0/3630407038 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mon_command_ack([{"prefix": "mgr stat"}]=0 v13) v1 ==== 56+0+104 (secure 0 0 0) 0x7feadc022720 con 0x7feae006fc90 2023-11-17T05:21:26.393 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: { 2023-11-17T05:21:26.393 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: "epoch": 13, 2023-11-17T05:21:26.393 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: "available": true, 2023-11-17T05:21:26.393 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: "active_name": "smithi040.ogcprl", 2023-11-17T05:21:26.393 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: "num_standby": 0 2023-11-17T05:21:26.393 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: } 2023-11-17T05:21:26.393 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:06.535+0000 7feae789e700 1 -- 172.21.15.40:0/3630407038 >> [v2:172.21.15.40:6800/1381511815,v1:172.21.15.40:6801/1381511815] conn(0x7feac802ff00 msgr2=0x7feac80323c0 unknown :-1 s=STATE_CONNECTING l=1).mark_down 2023-11-17T05:21:26.394 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:06.535+0000 7feae789e700 1 --2- 172.21.15.40:0/3630407038 >> [v2:172.21.15.40:6800/1381511815,v1:172.21.15.40:6801/1381511815] conn(0x7feac802ff00 0x7feac80323c0 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).stop 2023-11-17T05:21:26.394 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:06.535+0000 7feae789e700 1 -- 172.21.15.40:0/3630407038 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7feae006fc90 msgr2=0x7feae010a9f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:21:26.394 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:06.535+0000 7feae789e700 1 --2- 172.21.15.40:0/3630407038 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7feae006fc90 0x7feae010a9f0 secure :-1 s=READY pgs=73 cs=0 l=1 rev1=1 rx=0x7feadc000c00 tx=0x7feadc006ee0).stop 2023-11-17T05:21:26.394 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:06.536+0000 7feae789e700 1 -- 172.21.15.40:0/3630407038 shutdown_connections 2023-11-17T05:21:26.394 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:06.536+0000 7feae789e700 1 --2- 172.21.15.40:0/3630407038 >> [v2:172.21.15.40:6800/1381511815,v1:172.21.15.40:6801/1381511815] conn(0x7feac802ff00 0x7feac80323c0 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).stop 2023-11-17T05:21:26.394 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:06.536+0000 7feae789e700 1 --2- 172.21.15.40:0/3630407038 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7feae006fc90 0x7feae010a9f0 unknown :-1 s=CLOSED pgs=73 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:21:26.394 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:06.536+0000 7feae789e700 1 -- 172.21.15.40:0/3630407038 >> 172.21.15.40:0/3630407038 conn(0x7feae00f6280 msgr2=0x7feae00f7e70 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:21:26.394 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:06.536+0000 7feae789e700 1 -- 172.21.15.40:0/3630407038 shutdown_connections 2023-11-17T05:21:26.394 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:06.536+0000 7feae789e700 1 -- 172.21.15.40:0/3630407038 wait complete. 2023-11-17T05:21:26.394 INFO:teuthology.orchestra.run.smithi040.stderr:Waiting for the mgr to restart... 2023-11-17T05:21:26.394 INFO:teuthology.orchestra.run.smithi040.stderr:Waiting for mgr epoch 13... 2023-11-17T05:21:26.394 INFO:teuthology.orchestra.run.smithi040.stderr:Running command: /bin/podman run --rm --ipc=host --no-hosts --net=host --entrypoint /usr/bin/ceph --init -e CONTAINER_IMAGE=docker.io/ceph/ceph:v16.2.4 -e NODE_NAME=smithi040 -e CEPH_USE_RANDOM_NONCE=1 -v /var/log/ceph/ed6470e2-8508-11ee-8dbe-212e2dc638e7:/var/log/ceph:z -v /tmp/ceph-tmpzx79_7_h:/etc/ceph/ceph.client.admin.keyring:z -v /tmp/ceph-tmpru9tjirw:/etc/ceph/ceph.conf:z docker.io/ceph/ceph:v16.2.4 tell mgr mgr_status 2023-11-17T05:21:26.394 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:07.418+0000 7fa6889f0700 1 Processor -- start 2023-11-17T05:21:26.395 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:07.419+0000 7fa6889f0700 1 -- start start 2023-11-17T05:21:26.395 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:07.419+0000 7fa6889f0700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fa6840faae0 0x7fa6840faf00 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 rx=0 tx=0).connect 2023-11-17T05:21:26.395 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:07.419+0000 7fa6889f0700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fa6840fb480 con 0x7fa6840faae0 2023-11-17T05:21:26.395 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:07.419+0000 7fa68259c700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fa6840faae0 0x7fa6840faf00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:21:26.395 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:07.419+0000 7fa68259c700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fa6840faae0 0x7fa6840faf00 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:45656/0 (socket says 172.21.15.40:45656) 2023-11-17T05:21:26.395 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:07.419+0000 7fa68259c700 1 -- 172.21.15.40:0/1829280951 learned_addr learned my addr 172.21.15.40:0/1829280951 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-11-17T05:21:26.395 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:07.420+0000 7fa68259c700 1 -- 172.21.15.40:0/1829280951 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fa6840fb5c0 con 0x7fa6840faae0 2023-11-17T05:21:26.395 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:07.420+0000 7fa68259c700 1 --2- 172.21.15.40:0/1829280951 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fa6840faae0 0x7fa6840faf00 secure :-1 s=READY pgs=74 cs=0 l=1 rev1=1 rx=0x7fa67400b5f0 tx=0x7fa674006ee0).ready entity=mon.0 client_cookie=e3266a59518a03dc server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:21:26.395 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:07.420+0000 7fa68159a700 1 -- 172.21.15.40:0/1829280951 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7fa67400d600 con 0x7fa6840faae0 2023-11-17T05:21:26.395 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:07.420+0000 7fa68159a700 1 -- 172.21.15.40:0/1829280951 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(25 keys) v1 ==== 1010+0+0 (secure 0 0 0) 0x7fa67400dbd0 con 0x7fa6840faae0 2023-11-17T05:21:26.395 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:07.420+0000 7fa68159a700 1 -- 172.21.15.40:0/1829280951 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7fa674011c00 con 0x7fa6840faae0 2023-11-17T05:21:26.395 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:07.421+0000 7fa6889f0700 1 -- 172.21.15.40:0/1829280951 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fa6840faae0 msgr2=0x7fa6840faf00 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:21:26.395 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:07.421+0000 7fa6889f0700 1 --2- 172.21.15.40:0/1829280951 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fa6840faae0 0x7fa6840faf00 secure :-1 s=READY pgs=74 cs=0 l=1 rev1=1 rx=0x7fa67400b5f0 tx=0x7fa674006ee0).stop 2023-11-17T05:21:26.395 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:07.421+0000 7fa6889f0700 1 -- 172.21.15.40:0/1829280951 shutdown_connections 2023-11-17T05:21:26.396 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:07.421+0000 7fa6889f0700 1 --2- 172.21.15.40:0/1829280951 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fa6840faae0 0x7fa6840faf00 unknown :-1 s=CLOSED pgs=74 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:21:26.396 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:07.421+0000 7fa6889f0700 1 -- 172.21.15.40:0/1829280951 >> 172.21.15.40:0/1829280951 conn(0x7fa6840f6280 msgr2=0x7fa6840f86a0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:21:26.396 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:07.421+0000 7fa6889f0700 1 -- 172.21.15.40:0/1829280951 shutdown_connections 2023-11-17T05:21:26.396 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:07.421+0000 7fa6889f0700 1 -- 172.21.15.40:0/1829280951 wait complete. 2023-11-17T05:21:26.396 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:07.422+0000 7fa6889f0700 1 Processor -- start 2023-11-17T05:21:26.396 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:07.422+0000 7fa6889f0700 1 -- start start 2023-11-17T05:21:26.396 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:07.422+0000 7fa6889f0700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fa6840faae0 0x7fa6841087d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:21:26.396 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:07.422+0000 7fa6889f0700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fa684108cc0 con 0x7fa6840faae0 2023-11-17T05:21:26.396 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:07.422+0000 7fa68259c700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fa6840faae0 0x7fa6841087d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:21:26.396 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:07.422+0000 7fa68259c700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fa6840faae0 0x7fa6841087d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:45662/0 (socket says 172.21.15.40:45662) 2023-11-17T05:21:26.396 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:07.422+0000 7fa68259c700 1 -- 172.21.15.40:0/1834769551 learned_addr learned my addr 172.21.15.40:0/1834769551 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-11-17T05:21:26.396 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:07.423+0000 7fa68259c700 1 -- 172.21.15.40:0/1834769551 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fa67400b200 con 0x7fa6840faae0 2023-11-17T05:21:26.396 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:07.423+0000 7fa68259c700 1 --2- 172.21.15.40:0/1834769551 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fa6840faae0 0x7fa6841087d0 secure :-1 s=READY pgs=75 cs=0 l=1 rev1=1 rx=0x7fa67400b1d0 tx=0x7fa674006ee0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:21:26.397 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:07.423+0000 7fa672ffd700 1 -- 172.21.15.40:0/1834769551 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7fa674011a20 con 0x7fa6840faae0 2023-11-17T05:21:26.397 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:07.423+0000 7fa672ffd700 1 -- 172.21.15.40:0/1834769551 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(25 keys) v1 ==== 1010+0+0 (secure 0 0 0) 0x7fa67401b880 con 0x7fa6840faae0 2023-11-17T05:21:26.397 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:07.423+0000 7fa672ffd700 1 -- 172.21.15.40:0/1834769551 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7fa67401a700 con 0x7fa6840faae0 2023-11-17T05:21:26.397 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:07.423+0000 7fa6889f0700 1 -- 172.21.15.40:0/1834769551 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fa684108ec0 con 0x7fa6840faae0 2023-11-17T05:21:26.397 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:07.423+0000 7fa6889f0700 1 -- 172.21.15.40:0/1834769551 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fa6841092d0 con 0x7fa6840faae0 2023-11-17T05:21:26.397 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:07.424+0000 7fa672ffd700 1 -- 172.21.15.40:0/1834769551 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 13) v1 ==== 36208+0+0 (secure 0 0 0) 0x7fa67400f070 con 0x7fa6840faae0 2023-11-17T05:21:26.397 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:07.424+0000 7fa672ffd700 1 --2- 172.21.15.40:0/1834769551 >> [v2:172.21.15.40:6800/1381511815,v1:172.21.15.40:6801/1381511815] conn(0x7fa6680321a0 0x7fa668034660 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:21:26.397 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:07.424+0000 7fa672ffd700 1 -- 172.21.15.40:0/1834769551 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(3..3 src has 1..3) v4 ==== 1098+0+0 (secure 0 0 0) 0x7fa674041290 con 0x7fa6840faae0 2023-11-17T05:21:26.397 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:07.424+0000 7fa681d9b700 1 -- 172.21.15.40:0/1834769551 >> [v2:172.21.15.40:6800/1381511815,v1:172.21.15.40:6801/1381511815] conn(0x7fa6680321a0 msgr2=0x7fa668034660 unknown :-1 s=STATE_CONNECTING_RE l=1).process reconnect failed to v2:172.21.15.40:6800/1381511815 2023-11-17T05:21:26.397 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:07.424+0000 7fa681d9b700 1 --2- 172.21.15.40:0/1834769551 >> [v2:172.21.15.40:6800/1381511815,v1:172.21.15.40:6801/1381511815] conn(0x7fa6680321a0 0x7fa668034660 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._fault waiting 0.200000 2023-11-17T05:21:26.397 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:07.424+0000 7fa6889f0700 1 -- 172.21.15.40:0/1834769551 --> [v2:172.21.15.40:6800/1381511815,v1:172.21.15.40:6801/1381511815] -- command(tid 0: {"prefix": "get_command_descriptions"}) v1 -- 0x7fa684059100 con 0x7fa6680321a0 2023-11-17T05:21:26.397 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:07.625+0000 7fa681d9b700 1 -- 172.21.15.40:0/1834769551 >> [v2:172.21.15.40:6800/1381511815,v1:172.21.15.40:6801/1381511815] conn(0x7fa6680321a0 msgr2=0x7fa668034660 unknown :-1 s=STATE_CONNECTING_RE l=1).process reconnect failed to v2:172.21.15.40:6800/1381511815 2023-11-17T05:21:26.397 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:07.625+0000 7fa681d9b700 1 --2- 172.21.15.40:0/1834769551 >> [v2:172.21.15.40:6800/1381511815,v1:172.21.15.40:6801/1381511815] conn(0x7fa6680321a0 0x7fa668034660 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._fault waiting 0.400000 2023-11-17T05:21:26.397 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:08.026+0000 7fa681d9b700 1 -- 172.21.15.40:0/1834769551 >> [v2:172.21.15.40:6800/1381511815,v1:172.21.15.40:6801/1381511815] conn(0x7fa6680321a0 msgr2=0x7fa668034660 unknown :-1 s=STATE_CONNECTING_RE l=1).process reconnect failed to v2:172.21.15.40:6800/1381511815 2023-11-17T05:21:26.398 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:08.026+0000 7fa681d9b700 1 --2- 172.21.15.40:0/1834769551 >> [v2:172.21.15.40:6800/1381511815,v1:172.21.15.40:6801/1381511815] conn(0x7fa6680321a0 0x7fa668034660 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._fault waiting 0.800000 2023-11-17T05:21:26.398 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:08.827+0000 7fa681d9b700 1 -- 172.21.15.40:0/1834769551 >> [v2:172.21.15.40:6800/1381511815,v1:172.21.15.40:6801/1381511815] conn(0x7fa6680321a0 msgr2=0x7fa668034660 unknown :-1 s=STATE_CONNECTING_RE l=1).process reconnect failed to v2:172.21.15.40:6800/1381511815 2023-11-17T05:21:26.398 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:08.827+0000 7fa681d9b700 1 --2- 172.21.15.40:0/1834769551 >> [v2:172.21.15.40:6800/1381511815,v1:172.21.15.40:6801/1381511815] conn(0x7fa6680321a0 0x7fa668034660 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._fault waiting 1.600000 2023-11-17T05:21:26.398 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:10.428+0000 7fa681d9b700 1 -- 172.21.15.40:0/1834769551 >> [v2:172.21.15.40:6800/1381511815,v1:172.21.15.40:6801/1381511815] conn(0x7fa6680321a0 msgr2=0x7fa668034660 unknown :-1 s=STATE_CONNECTING_RE l=1).process reconnect failed to v2:172.21.15.40:6800/1381511815 2023-11-17T05:21:26.398 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:10.428+0000 7fa681d9b700 1 --2- 172.21.15.40:0/1834769551 >> [v2:172.21.15.40:6800/1381511815,v1:172.21.15.40:6801/1381511815] conn(0x7fa6680321a0 0x7fa668034660 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._fault waiting 3.200000 2023-11-17T05:21:26.398 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:11.149+0000 7fa672ffd700 1 -- 172.21.15.40:0/1834769551 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mgrmap(e 14) v1 ==== 35997+0+0 (secure 0 0 0) 0x7fa674025500 con 0x7fa6840faae0 2023-11-17T05:21:26.398 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:11.149+0000 7fa672ffd700 1 -- 172.21.15.40:0/1834769551 >> [v2:172.21.15.40:6800/1381511815,v1:172.21.15.40:6801/1381511815] conn(0x7fa6680321a0 msgr2=0x7fa668034660 unknown :-1 s=STATE_CONNECTING l=1).mark_down 2023-11-17T05:21:26.398 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:11.149+0000 7fa672ffd700 1 --2- 172.21.15.40:0/1834769551 >> [v2:172.21.15.40:6800/1381511815,v1:172.21.15.40:6801/1381511815] conn(0x7fa6680321a0 0x7fa668034660 unknown :-1 s=START_CONNECT pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).stop 2023-11-17T05:21:26.398 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:12.154+0000 7fa672ffd700 1 -- 172.21.15.40:0/1834769551 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mgrmap(e 15) v1 ==== 36067+0+0 (secure 0 0 0) 0x7fa674040ee0 con 0x7fa6840faae0 2023-11-17T05:21:26.398 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:12.154+0000 7fa672ffd700 1 --2- 172.21.15.40:0/1834769551 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fa668035c80 0x7fa668038070 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:21:26.398 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:12.154+0000 7fa672ffd700 1 -- 172.21.15.40:0/1834769551 --> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] -- command(tid 0: {"prefix": "get_command_descriptions"}) v1 -- 0x7fa684059100 con 0x7fa668035c80 2023-11-17T05:21:26.398 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:12.155+0000 7fa681d9b700 1 --2- 172.21.15.40:0/1834769551 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fa668035c80 0x7fa668038070 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:21:26.398 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:12.156+0000 7fa681d9b700 1 --2- 172.21.15.40:0/1834769551 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fa668035c80 0x7fa668038070 secure :-1 s=READY pgs=2 cs=0 l=1 rev1=1 rx=0x7fa678006fd0 tx=0x7fa678006df0).ready entity=mgr.14164 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:21:26.399 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:12.156+0000 7fa672ffd700 1 -- 172.21.15.40:0/1834769551 <== mgr.14164 v2:172.21.15.40:6800/1119389001 1 ==== command_reply(tid 0: 0 ) v1 ==== 8+0+6438 (secure 0 0 0) 0x7fa684059100 con 0x7fa668035c80 2023-11-17T05:21:26.399 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:12.164+0000 7fa6889f0700 1 -- 172.21.15.40:0/1834769551 --> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] -- command(tid 1: {"prefix": "mgr_status"}) v1 -- 0x7fa684045ad0 con 0x7fa668035c80 2023-11-17T05:21:26.399 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:12.164+0000 7fa672ffd700 1 -- 172.21.15.40:0/1834769551 <== mgr.14164 v2:172.21.15.40:6800/1119389001 2 ==== command_reply(tid 1: 0 ) v1 ==== 8+0+52 (secure 0 0 0) 0x7fa684045ad0 con 0x7fa668035c80 2023-11-17T05:21:26.399 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: { 2023-11-17T05:21:26.399 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: "mgrmap_epoch": 15, 2023-11-17T05:21:26.399 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: "initialized": true 2023-11-17T05:21:26.399 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: } 2023-11-17T05:21:26.399 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:12.165+0000 7fa6889f0700 1 -- 172.21.15.40:0/1834769551 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fa668035c80 msgr2=0x7fa668038070 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:21:26.400 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:12.165+0000 7fa6889f0700 1 --2- 172.21.15.40:0/1834769551 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fa668035c80 0x7fa668038070 secure :-1 s=READY pgs=2 cs=0 l=1 rev1=1 rx=0x7fa678006fd0 tx=0x7fa678006df0).stop 2023-11-17T05:21:26.400 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:12.165+0000 7fa6889f0700 1 -- 172.21.15.40:0/1834769551 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fa6840faae0 msgr2=0x7fa6841087d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:21:26.400 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:12.165+0000 7fa6889f0700 1 --2- 172.21.15.40:0/1834769551 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fa6840faae0 0x7fa6841087d0 secure :-1 s=READY pgs=75 cs=0 l=1 rev1=1 rx=0x7fa67400b1d0 tx=0x7fa674006ee0).stop 2023-11-17T05:21:26.400 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:12.165+0000 7fa6889f0700 1 -- 172.21.15.40:0/1834769551 shutdown_connections 2023-11-17T05:21:26.401 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:12.165+0000 7fa6889f0700 1 --2- 172.21.15.40:0/1834769551 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fa668035c80 0x7fa668038070 unknown :-1 s=CLOSED pgs=2 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:21:26.401 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:12.165+0000 7fa6889f0700 1 --2- 172.21.15.40:0/1834769551 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fa6840faae0 0x7fa6841087d0 unknown :-1 s=CLOSED pgs=75 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:21:26.401 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:12.165+0000 7fa6889f0700 1 --2- 172.21.15.40:0/1834769551 >> [v2:172.21.15.40:6800/1381511815,v1:172.21.15.40:6801/1381511815] conn(0x7fa6680321a0 0x7fa668034660 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).stop 2023-11-17T05:21:26.401 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:12.165+0000 7fa6889f0700 1 -- 172.21.15.40:0/1834769551 >> 172.21.15.40:0/1834769551 conn(0x7fa6840f6280 msgr2=0x7fa6840ff7d0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:21:26.401 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:12.165+0000 7fa6889f0700 1 -- 172.21.15.40:0/1834769551 shutdown_connections 2023-11-17T05:21:26.401 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:12.165+0000 7fa6889f0700 1 -- 172.21.15.40:0/1834769551 wait complete. 2023-11-17T05:21:26.401 INFO:teuthology.orchestra.run.smithi040.stderr:mgr epoch 13 is available 2023-11-17T05:21:26.401 INFO:teuthology.orchestra.run.smithi040.stderr:Generating a dashboard self-signed certificate... 2023-11-17T05:21:26.401 INFO:teuthology.orchestra.run.smithi040.stderr:Running command: /bin/podman run --rm --ipc=host --no-hosts --net=host --entrypoint /usr/bin/ceph --init -e CONTAINER_IMAGE=docker.io/ceph/ceph:v16.2.4 -e NODE_NAME=smithi040 -e CEPH_USE_RANDOM_NONCE=1 -v /var/log/ceph/ed6470e2-8508-11ee-8dbe-212e2dc638e7:/var/log/ceph:z -v /tmp/ceph-tmpzx79_7_h:/etc/ceph/ceph.client.admin.keyring:z -v /tmp/ceph-tmpru9tjirw:/etc/ceph/ceph.conf:z docker.io/ceph/ceph:v16.2.4 dashboard create-self-signed-cert 2023-11-17T05:21:26.402 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:12.963+0000 7f0bd79ca700 1 Processor -- start 2023-11-17T05:21:26.402 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:12.963+0000 7f0bd79ca700 1 -- start start 2023-11-17T05:21:26.402 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:12.963+0000 7f0bd79ca700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f0bd006fc90 0x7f0bd00fe7e0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 rx=0 tx=0).connect 2023-11-17T05:21:26.402 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:12.963+0000 7f0bd79ca700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f0bd00fed60 con 0x7f0bd006fc90 2023-11-17T05:21:26.402 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:12.964+0000 7f0bd5766700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f0bd006fc90 0x7f0bd00fe7e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:21:26.402 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:12.964+0000 7f0bd5766700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f0bd006fc90 0x7f0bd00fe7e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:45724/0 (socket says 172.21.15.40:45724) 2023-11-17T05:21:26.402 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:12.964+0000 7f0bd5766700 1 -- 172.21.15.40:0/3125744453 learned_addr learned my addr 172.21.15.40:0/3125744453 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-11-17T05:21:26.402 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:12.964+0000 7f0bd5766700 1 -- 172.21.15.40:0/3125744453 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f0bd00feea0 con 0x7f0bd006fc90 2023-11-17T05:21:26.402 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:12.964+0000 7f0bd5766700 1 --2- 172.21.15.40:0/3125744453 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f0bd006fc90 0x7f0bd00fe7e0 secure :-1 s=READY pgs=81 cs=0 l=1 rev1=1 rx=0x7f0bcc00b5f0 tx=0x7f0bcc006ee0).ready entity=mon.0 client_cookie=89ce71bb79e22b99 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:21:26.402 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:12.965+0000 7f0bbffff700 1 -- 172.21.15.40:0/3125744453 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f0bcc00d600 con 0x7f0bd006fc90 2023-11-17T05:21:26.402 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:12.965+0000 7f0bbffff700 1 -- 172.21.15.40:0/3125744453 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(25 keys) v1 ==== 1010+0+0 (secure 0 0 0) 0x7f0bcc00dbd0 con 0x7f0bd006fc90 2023-11-17T05:21:26.402 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:12.965+0000 7f0bbffff700 1 -- 172.21.15.40:0/3125744453 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f0bcc011c00 con 0x7f0bd006fc90 2023-11-17T05:21:26.403 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:12.965+0000 7f0bd79ca700 1 -- 172.21.15.40:0/3125744453 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f0bd006fc90 msgr2=0x7f0bd00fe7e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:21:26.403 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:12.965+0000 7f0bd79ca700 1 --2- 172.21.15.40:0/3125744453 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f0bd006fc90 0x7f0bd00fe7e0 secure :-1 s=READY pgs=81 cs=0 l=1 rev1=1 rx=0x7f0bcc00b5f0 tx=0x7f0bcc006ee0).stop 2023-11-17T05:21:26.403 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:12.965+0000 7f0bd79ca700 1 -- 172.21.15.40:0/3125744453 shutdown_connections 2023-11-17T05:21:26.403 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:12.965+0000 7f0bd79ca700 1 --2- 172.21.15.40:0/3125744453 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f0bd006fc90 0x7f0bd00fe7e0 unknown :-1 s=CLOSED pgs=81 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:21:26.403 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:12.965+0000 7f0bd79ca700 1 -- 172.21.15.40:0/3125744453 >> 172.21.15.40:0/3125744453 conn(0x7f0bd00f6280 msgr2=0x7f0bd00f86a0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:21:26.403 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:12.966+0000 7f0bd79ca700 1 -- 172.21.15.40:0/3125744453 shutdown_connections 2023-11-17T05:21:26.403 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:12.966+0000 7f0bd79ca700 1 -- 172.21.15.40:0/3125744453 wait complete. 2023-11-17T05:21:26.403 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:12.966+0000 7f0bd79ca700 1 Processor -- start 2023-11-17T05:21:26.403 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:12.966+0000 7f0bd79ca700 1 -- start start 2023-11-17T05:21:26.403 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:12.966+0000 7f0bd79ca700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f0bd006fc90 0x7f0bd010a9a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:21:26.403 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:12.966+0000 7f0bd79ca700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f0bd010ae90 con 0x7f0bd006fc90 2023-11-17T05:21:26.403 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:12.967+0000 7f0bd5766700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f0bd006fc90 0x7f0bd010a9a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:21:26.403 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:12.967+0000 7f0bd5766700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f0bd006fc90 0x7f0bd010a9a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:45732/0 (socket says 172.21.15.40:45732) 2023-11-17T05:21:26.404 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:12.967+0000 7f0bd5766700 1 -- 172.21.15.40:0/2903078684 learned_addr learned my addr 172.21.15.40:0/2903078684 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-11-17T05:21:26.404 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:12.967+0000 7f0bd5766700 1 -- 172.21.15.40:0/2903078684 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f0bcc00b200 con 0x7f0bd006fc90 2023-11-17T05:21:26.404 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:12.967+0000 7f0bd5766700 1 --2- 172.21.15.40:0/2903078684 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f0bd006fc90 0x7f0bd010a9a0 secure :-1 s=READY pgs=82 cs=0 l=1 rev1=1 rx=0x7f0bcc00e040 tx=0x7f0bcc006ee0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:21:26.404 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:12.967+0000 7f0bbdffb700 1 -- 172.21.15.40:0/2903078684 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f0bcc01b410 con 0x7f0bd006fc90 2023-11-17T05:21:26.404 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:12.967+0000 7f0bbdffb700 1 -- 172.21.15.40:0/2903078684 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(25 keys) v1 ==== 1010+0+0 (secure 0 0 0) 0x7f0bcc01b9e0 con 0x7f0bd006fc90 2023-11-17T05:21:26.404 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:12.967+0000 7f0bd79ca700 1 -- 172.21.15.40:0/2903078684 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f0bd010b090 con 0x7f0bd006fc90 2023-11-17T05:21:26.404 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:12.967+0000 7f0bd79ca700 1 -- 172.21.15.40:0/2903078684 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f0bd010b4a0 con 0x7f0bd006fc90 2023-11-17T05:21:26.404 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:12.968+0000 7f0bbdffb700 1 -- 172.21.15.40:0/2903078684 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f0bcc01a770 con 0x7f0bd006fc90 2023-11-17T05:21:26.404 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:12.968+0000 7f0bbdffb700 1 -- 172.21.15.40:0/2903078684 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 15) v1 ==== 36067+0+0 (secure 0 0 0) 0x7f0bcc00f070 con 0x7f0bd006fc90 2023-11-17T05:21:26.404 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:12.969+0000 7f0bbdffb700 1 --2- 172.21.15.40:0/2903078684 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f0bb802fdf0 0x7f0bb80322b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:21:26.404 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:12.969+0000 7f0bbdffb700 1 -- 172.21.15.40:0/2903078684 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(4..4 src has 1..4) v4 ==== 1270+0+0 (secure 0 0 0) 0x7f0bcc041f60 con 0x7f0bd006fc90 2023-11-17T05:21:26.404 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:12.969+0000 7f0bd4f65700 1 --2- 172.21.15.40:0/2903078684 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f0bb802fdf0 0x7f0bb80322b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:21:26.405 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:12.969+0000 7f0bd79ca700 1 -- 172.21.15.40:0/2903078684 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f0bd0045ad0 con 0x7f0bd006fc90 2023-11-17T05:21:26.405 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:12.970+0000 7f0bd4f65700 1 --2- 172.21.15.40:0/2903078684 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f0bb802fdf0 0x7f0bb80322b0 secure :-1 s=READY pgs=6 cs=0 l=1 rev1=1 rx=0x7f0bc00096a0 tx=0x7f0bc0006b40).ready entity=mgr.14164 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:21:26.405 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:12.974+0000 7f0bbdffb700 1 -- 172.21.15.40:0/2903078684 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+152212 (secure 0 0 0) 0x7f0bcc0682b0 con 0x7f0bd006fc90 2023-11-17T05:21:26.405 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:13.143+0000 7f0bd79ca700 1 -- 172.21.15.40:0/2903078684 --> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] -- mgr_command(tid 0: {"prefix": "dashboard create-self-signed-cert", "target": ["mon-mgr", ""]}) v1 -- 0x7f0bd01048d0 con 0x7f0bb802fdf0 2023-11-17T05:21:26.405 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:13.162+0000 7f0bbdffb700 1 -- 172.21.15.40:0/2903078684 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mgrmap(e 16) v1 ==== 36208+0+0 (secure 0 0 0) 0x7f0bcc024790 con 0x7f0bd006fc90 2023-11-17T05:21:26.405 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:13.181+0000 7f0bbdffb700 1 -- 172.21.15.40:0/2903078684 <== mgr.14164 v2:172.21.15.40:6800/1119389001 1 ==== mgr_command_reply(tid 0: 0 ) v1 ==== 8+0+32 (secure 0 0 0) 0x7f0bd01048d0 con 0x7f0bb802fdf0 2023-11-17T05:21:26.405 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: Self-signed certificate created 2023-11-17T05:21:26.405 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:13.183+0000 7f0bd79ca700 1 -- 172.21.15.40:0/2903078684 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f0bb802fdf0 msgr2=0x7f0bb80322b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:21:26.405 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:13.183+0000 7f0bd79ca700 1 --2- 172.21.15.40:0/2903078684 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f0bb802fdf0 0x7f0bb80322b0 secure :-1 s=READY pgs=6 cs=0 l=1 rev1=1 rx=0x7f0bc00096a0 tx=0x7f0bc0006b40).stop 2023-11-17T05:21:26.405 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:13.183+0000 7f0bd79ca700 1 -- 172.21.15.40:0/2903078684 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f0bd006fc90 msgr2=0x7f0bd010a9a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:21:26.405 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:13.183+0000 7f0bd79ca700 1 --2- 172.21.15.40:0/2903078684 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f0bd006fc90 0x7f0bd010a9a0 secure :-1 s=READY pgs=82 cs=0 l=1 rev1=1 rx=0x7f0bcc00e040 tx=0x7f0bcc006ee0).stop 2023-11-17T05:21:26.405 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:13.183+0000 7f0bd79ca700 1 -- 172.21.15.40:0/2903078684 shutdown_connections 2023-11-17T05:21:26.406 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:13.183+0000 7f0bd79ca700 1 --2- 172.21.15.40:0/2903078684 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f0bb802fdf0 0x7f0bb80322b0 unknown :-1 s=CLOSED pgs=6 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:21:26.406 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:13.183+0000 7f0bd79ca700 1 --2- 172.21.15.40:0/2903078684 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f0bd006fc90 0x7f0bd010a9a0 unknown :-1 s=CLOSED pgs=82 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:21:26.406 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:13.184+0000 7f0bd79ca700 1 -- 172.21.15.40:0/2903078684 >> 172.21.15.40:0/2903078684 conn(0x7f0bd00f6280 msgr2=0x7f0bd00f7e50 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:21:26.406 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:13.185+0000 7f0bd79ca700 1 -- 172.21.15.40:0/2903078684 shutdown_connections 2023-11-17T05:21:26.406 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:13.185+0000 7f0bd79ca700 1 -- 172.21.15.40:0/2903078684 wait complete. 2023-11-17T05:21:26.406 INFO:teuthology.orchestra.run.smithi040.stderr:Creating initial admin user... 2023-11-17T05:21:26.406 INFO:teuthology.orchestra.run.smithi040.stderr:Running command: /bin/podman run --rm --ipc=host --no-hosts --net=host --entrypoint /usr/bin/ceph --init -e CONTAINER_IMAGE=docker.io/ceph/ceph:v16.2.4 -e NODE_NAME=smithi040 -e CEPH_USE_RANDOM_NONCE=1 -v /var/log/ceph/ed6470e2-8508-11ee-8dbe-212e2dc638e7:/var/log/ceph:z -v /tmp/ceph-tmpzx79_7_h:/etc/ceph/ceph.client.admin.keyring:z -v /tmp/ceph-tmpru9tjirw:/etc/ceph/ceph.conf:z -v /tmp/ceph-tmpzyup_tl3:/tmp/dashboard.pw:z docker.io/ceph/ceph:v16.2.4 dashboard ac-user-create admin -i /tmp/dashboard.pw administrator --force-password --pwd-update-required 2023-11-17T05:21:26.406 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:13.988+0000 7f7e873d9700 1 Processor -- start 2023-11-17T05:21:26.406 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:13.988+0000 7f7e873d9700 1 -- start start 2023-11-17T05:21:26.406 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:13.988+0000 7f7e873d9700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f7e8006fcd0 0x7f7e800fe7a0 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 rx=0 tx=0).connect 2023-11-17T05:21:26.406 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:13.988+0000 7f7e873d9700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f7e800fed20 con 0x7f7e8006fcd0 2023-11-17T05:21:26.406 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:13.988+0000 7f7e85175700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f7e8006fcd0 0x7f7e800fe7a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:21:26.407 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:13.989+0000 7f7e85175700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f7e8006fcd0 0x7f7e800fe7a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:45748/0 (socket says 172.21.15.40:45748) 2023-11-17T05:21:26.407 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:13.989+0000 7f7e85175700 1 -- 172.21.15.40:0/1573259514 learned_addr learned my addr 172.21.15.40:0/1573259514 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-11-17T05:21:26.407 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:13.989+0000 7f7e85175700 1 -- 172.21.15.40:0/1573259514 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f7e800fee60 con 0x7f7e8006fcd0 2023-11-17T05:21:26.407 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:13.989+0000 7f7e85175700 1 --2- 172.21.15.40:0/1573259514 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f7e8006fcd0 0x7f7e800fe7a0 secure :-1 s=READY pgs=83 cs=0 l=1 rev1=1 rx=0x7f7e7000b5f0 tx=0x7f7e70006ee0).ready entity=mon.0 client_cookie=e20dc9dbe64b65f5 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:21:26.407 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:13.989+0000 7f7e6ffff700 1 -- 172.21.15.40:0/1573259514 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f7e7000d600 con 0x7f7e8006fcd0 2023-11-17T05:21:26.407 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:13.989+0000 7f7e6ffff700 1 -- 172.21.15.40:0/1573259514 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(25 keys) v1 ==== 1010+0+0 (secure 0 0 0) 0x7f7e7000dbd0 con 0x7f7e8006fcd0 2023-11-17T05:21:26.407 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:13.989+0000 7f7e6ffff700 1 -- 172.21.15.40:0/1573259514 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f7e70011c00 con 0x7f7e8006fcd0 2023-11-17T05:21:26.407 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:13.990+0000 7f7e873d9700 1 -- 172.21.15.40:0/1573259514 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f7e8006fcd0 msgr2=0x7f7e800fe7a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:21:26.407 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:13.990+0000 7f7e873d9700 1 --2- 172.21.15.40:0/1573259514 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f7e8006fcd0 0x7f7e800fe7a0 secure :-1 s=READY pgs=83 cs=0 l=1 rev1=1 rx=0x7f7e7000b5f0 tx=0x7f7e70006ee0).stop 2023-11-17T05:21:26.407 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:13.990+0000 7f7e873d9700 1 -- 172.21.15.40:0/1573259514 shutdown_connections 2023-11-17T05:21:26.407 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:13.990+0000 7f7e873d9700 1 --2- 172.21.15.40:0/1573259514 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f7e8006fcd0 0x7f7e800fe7a0 unknown :-1 s=CLOSED pgs=83 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:21:26.408 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:13.990+0000 7f7e873d9700 1 -- 172.21.15.40:0/1573259514 >> 172.21.15.40:0/1573259514 conn(0x7f7e800f6280 msgr2=0x7f7e800f86a0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:21:26.408 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:13.990+0000 7f7e873d9700 1 -- 172.21.15.40:0/1573259514 shutdown_connections 2023-11-17T05:21:26.408 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:13.990+0000 7f7e873d9700 1 -- 172.21.15.40:0/1573259514 wait complete. 2023-11-17T05:21:26.408 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:13.991+0000 7f7e873d9700 1 Processor -- start 2023-11-17T05:21:26.408 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:13.991+0000 7f7e873d9700 1 -- start start 2023-11-17T05:21:26.409 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:13.991+0000 7f7e873d9700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f7e8006fcd0 0x7f7e8010a9b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:21:26.409 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:13.991+0000 7f7e873d9700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f7e8010aea0 con 0x7f7e8006fcd0 2023-11-17T05:21:26.409 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:13.991+0000 7f7e85175700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f7e8006fcd0 0x7f7e8010a9b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:21:26.410 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:13.991+0000 7f7e85175700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f7e8006fcd0 0x7f7e8010a9b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:45760/0 (socket says 172.21.15.40:45760) 2023-11-17T05:21:26.410 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:13.991+0000 7f7e85175700 1 -- 172.21.15.40:0/369202895 learned_addr learned my addr 172.21.15.40:0/369202895 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-11-17T05:21:26.410 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:13.992+0000 7f7e85175700 1 -- 172.21.15.40:0/369202895 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f7e7000b200 con 0x7f7e8006fcd0 2023-11-17T05:21:26.410 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:13.992+0000 7f7e85175700 1 --2- 172.21.15.40:0/369202895 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f7e8006fcd0 0x7f7e8010a9b0 secure :-1 s=READY pgs=84 cs=0 l=1 rev1=1 rx=0x7f7e7000e040 tx=0x7f7e70006ee0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:21:26.410 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:13.992+0000 7f7e6dffb700 1 -- 172.21.15.40:0/369202895 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f7e7001b410 con 0x7f7e8006fcd0 2023-11-17T05:21:26.410 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:13.992+0000 7f7e6dffb700 1 -- 172.21.15.40:0/369202895 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(25 keys) v1 ==== 1010+0+0 (secure 0 0 0) 0x7f7e7001b9e0 con 0x7f7e8006fcd0 2023-11-17T05:21:26.410 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:13.992+0000 7f7e873d9700 1 -- 172.21.15.40:0/369202895 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f7e8010b0a0 con 0x7f7e8006fcd0 2023-11-17T05:21:26.410 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:13.992+0000 7f7e6dffb700 1 -- 172.21.15.40:0/369202895 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f7e7001a770 con 0x7f7e8006fcd0 2023-11-17T05:21:26.410 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:13.992+0000 7f7e873d9700 1 -- 172.21.15.40:0/369202895 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f7e8010b4b0 con 0x7f7e8006fcd0 2023-11-17T05:21:26.410 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:13.993+0000 7f7e873d9700 1 -- 172.21.15.40:0/369202895 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f7e80045ad0 con 0x7f7e8006fcd0 2023-11-17T05:21:26.410 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:13.994+0000 7f7e6dffb700 1 -- 172.21.15.40:0/369202895 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 16) v1 ==== 36208+0+0 (secure 0 0 0) 0x7f7e7000f070 con 0x7f7e8006fcd0 2023-11-17T05:21:26.411 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:13.994+0000 7f7e6dffb700 1 --2- 172.21.15.40:0/369202895 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f7e6802ff00 0x7f7e680323c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:21:26.411 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:13.994+0000 7f7e6dffb700 1 -- 172.21.15.40:0/369202895 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(4..4 src has 1..4) v4 ==== 1270+0+0 (secure 0 0 0) 0x7f7e700418e0 con 0x7f7e8006fcd0 2023-11-17T05:21:26.411 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:13.994+0000 7f7e84974700 1 --2- 172.21.15.40:0/369202895 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f7e6802ff00 0x7f7e680323c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:21:26.412 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:13.995+0000 7f7e84974700 1 --2- 172.21.15.40:0/369202895 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f7e6802ff00 0x7f7e680323c0 secure :-1 s=READY pgs=7 cs=0 l=1 rev1=1 rx=0x7f7e7c0096a0 tx=0x7f7e7c006b40).ready entity=mgr.14164 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:21:26.412 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:13.998+0000 7f7e6dffb700 1 -- 172.21.15.40:0/369202895 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+152212 (secure 0 0 0) 0x7f7e70022b30 con 0x7f7e8006fcd0 2023-11-17T05:21:26.412 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:14.180+0000 7f7e873d9700 1 -- 172.21.15.40:0/369202895 --> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] -- mgr_command(tid 0: {"prefix": "dashboard ac-user-create", "username": "admin", "rolename": "administrator", "force_password": true, "pwd_update_required": true, "target": ["mon-mgr", ""]}) v1 -- 0x7f7e80104a30 con 0x7f7e6802ff00 2023-11-17T05:21:26.412 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:14.432+0000 7f7e6dffb700 1 -- 172.21.15.40:0/369202895 <== mgr.14164 v2:172.21.15.40:6800/1119389001 1 ==== mgr_command_reply(tid 0: 0 ) v1 ==== 8+0+252 (secure 0 0 0) 0x7f7e80104a30 con 0x7f7e6802ff00 2023-11-17T05:21:26.412 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: {"username": "admin", "password": "$2b$12$sMBm6kJ2m0OrXMIzSs2YsuHWU4XuCKMIQMjvLY71oNyoIFXbHXFF6", "roles": ["administrator"], "name": null, "email": null, "lastUpdate": 1700198474, "enabled": true, "pwdExpirationDate": null, "pwdUpdateRequired": true} 2023-11-17T05:21:26.412 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:14.434+0000 7f7e873d9700 1 -- 172.21.15.40:0/369202895 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f7e6802ff00 msgr2=0x7f7e680323c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:21:26.412 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:14.434+0000 7f7e873d9700 1 --2- 172.21.15.40:0/369202895 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f7e6802ff00 0x7f7e680323c0 secure :-1 s=READY pgs=7 cs=0 l=1 rev1=1 rx=0x7f7e7c0096a0 tx=0x7f7e7c006b40).stop 2023-11-17T05:21:26.412 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:14.434+0000 7f7e873d9700 1 -- 172.21.15.40:0/369202895 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f7e8006fcd0 msgr2=0x7f7e8010a9b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:21:26.412 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:14.434+0000 7f7e873d9700 1 --2- 172.21.15.40:0/369202895 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f7e8006fcd0 0x7f7e8010a9b0 secure :-1 s=READY pgs=84 cs=0 l=1 rev1=1 rx=0x7f7e7000e040 tx=0x7f7e70006ee0).stop 2023-11-17T05:21:26.413 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:14.434+0000 7f7e873d9700 1 -- 172.21.15.40:0/369202895 shutdown_connections 2023-11-17T05:21:26.413 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:14.434+0000 7f7e873d9700 1 --2- 172.21.15.40:0/369202895 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f7e6802ff00 0x7f7e680323c0 unknown :-1 s=CLOSED pgs=7 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:21:26.413 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:14.434+0000 7f7e873d9700 1 --2- 172.21.15.40:0/369202895 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f7e8006fcd0 0x7f7e8010a9b0 unknown :-1 s=CLOSED pgs=84 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:21:26.413 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:14.434+0000 7f7e873d9700 1 -- 172.21.15.40:0/369202895 >> 172.21.15.40:0/369202895 conn(0x7f7e800f6280 msgr2=0x7f7e800f7e60 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:21:26.413 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:14.434+0000 7f7e873d9700 1 -- 172.21.15.40:0/369202895 shutdown_connections 2023-11-17T05:21:26.413 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:14.434+0000 7f7e873d9700 1 -- 172.21.15.40:0/369202895 wait complete. 2023-11-17T05:21:26.413 INFO:teuthology.orchestra.run.smithi040.stderr:Fetching dashboard port number... 2023-11-17T05:21:26.413 INFO:teuthology.orchestra.run.smithi040.stderr:Running command: /bin/podman run --rm --ipc=host --no-hosts --net=host --entrypoint /usr/bin/ceph --init -e CONTAINER_IMAGE=docker.io/ceph/ceph:v16.2.4 -e NODE_NAME=smithi040 -e CEPH_USE_RANDOM_NONCE=1 -v /var/log/ceph/ed6470e2-8508-11ee-8dbe-212e2dc638e7:/var/log/ceph:z -v /tmp/ceph-tmpzx79_7_h:/etc/ceph/ceph.client.admin.keyring:z -v /tmp/ceph-tmpru9tjirw:/etc/ceph/ceph.conf:z docker.io/ceph/ceph:v16.2.4 config get mgr mgr/dashboard/ssl_server_port 2023-11-17T05:21:26.413 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:15.172+0000 7f3d5c643700 1 Processor -- start 2023-11-17T05:21:26.413 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:15.172+0000 7f3d5c643700 1 -- start start 2023-11-17T05:21:26.413 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:15.172+0000 7f3d5c643700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f3d54071cf0 0x7f3d54070290 unknown :-1 s=NONE pgs=0 cs=0 l=0 rev1=0 rx=0 tx=0).connect 2023-11-17T05:21:26.413 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:15.173+0000 7f3d5c643700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f3d54072110 con 0x7f3d54071cf0 2023-11-17T05:21:26.413 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:15.173+0000 7f3d5a3df700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f3d54071cf0 0x7f3d54070290 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=0 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:21:26.414 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:15.173+0000 7f3d5a3df700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f3d54071cf0 0x7f3d54070290 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=0 rev1=1 rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:43482/0 (socket says 172.21.15.40:43482) 2023-11-17T05:21:26.414 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:15.173+0000 7f3d5a3df700 1 -- 172.21.15.40:0/3862489893 learned_addr learned my addr 172.21.15.40:0/3862489893 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-11-17T05:21:26.414 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:15.173+0000 7f3d5a3df700 1 -- 172.21.15.40:0/3862489893 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f3d54070780 con 0x7f3d54071cf0 2023-11-17T05:21:26.415 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:15.173+0000 7f3d5a3df700 1 --2- 172.21.15.40:0/3862489893 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f3d54071cf0 0x7f3d54070290 secure :-1 s=READY pgs=85 cs=0 l=1 rev1=1 rx=0x7f3d4400b5f0 tx=0x7f3d44006ee0).ready entity=mon.0 client_cookie=4ce0e6944f01f9e2 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:21:26.415 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:15.173+0000 7f3d593dd700 1 -- 172.21.15.40:0/3862489893 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f3d4400d600 con 0x7f3d54071cf0 2023-11-17T05:21:26.415 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:15.173+0000 7f3d593dd700 1 -- 172.21.15.40:0/3862489893 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(25 keys) v1 ==== 1010+0+0 (secure 0 0 0) 0x7f3d4400dbd0 con 0x7f3d54071cf0 2023-11-17T05:21:26.415 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:15.173+0000 7f3d593dd700 1 -- 172.21.15.40:0/3862489893 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f3d44011c00 con 0x7f3d54071cf0 2023-11-17T05:21:26.415 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:15.174+0000 7f3d5c643700 1 -- 172.21.15.40:0/3862489893 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f3d54071cf0 msgr2=0x7f3d54070290 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:21:26.415 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:15.174+0000 7f3d5c643700 1 --2- 172.21.15.40:0/3862489893 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f3d54071cf0 0x7f3d54070290 secure :-1 s=READY pgs=85 cs=0 l=1 rev1=1 rx=0x7f3d4400b5f0 tx=0x7f3d44006ee0).stop 2023-11-17T05:21:26.415 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:15.174+0000 7f3d5c643700 1 -- 172.21.15.40:0/3862489893 shutdown_connections 2023-11-17T05:21:26.415 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:15.174+0000 7f3d5c643700 1 --2- 172.21.15.40:0/3862489893 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f3d54071cf0 0x7f3d54070290 unknown :-1 s=CLOSED pgs=85 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:21:26.416 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:15.174+0000 7f3d5c643700 1 -- 172.21.15.40:0/3862489893 >> 172.21.15.40:0/3862489893 conn(0x7f3d540f73f0 msgr2=0x7f3d540f9810 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:21:26.416 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:15.175+0000 7f3d5c643700 1 -- 172.21.15.40:0/3862489893 shutdown_connections 2023-11-17T05:21:26.416 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:15.175+0000 7f3d5c643700 1 -- 172.21.15.40:0/3862489893 wait complete. 2023-11-17T05:21:26.416 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:15.175+0000 7f3d5c643700 1 Processor -- start 2023-11-17T05:21:26.416 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:15.175+0000 7f3d5c643700 1 -- start start 2023-11-17T05:21:26.416 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:15.175+0000 7f3d5c643700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f3d54071cf0 0x7f3d54110fb0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:21:26.416 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:15.175+0000 7f3d5c643700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f3d541114a0 con 0x7f3d54071cf0 2023-11-17T05:21:26.416 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:15.176+0000 7f3d5a3df700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f3d54071cf0 0x7f3d54110fb0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:21:26.416 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:15.176+0000 7f3d5a3df700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f3d54071cf0 0x7f3d54110fb0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:43496/0 (socket says 172.21.15.40:43496) 2023-11-17T05:21:26.416 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:15.176+0000 7f3d5a3df700 1 -- 172.21.15.40:0/1923721708 learned_addr learned my addr 172.21.15.40:0/1923721708 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-11-17T05:21:26.416 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:15.176+0000 7f3d5a3df700 1 -- 172.21.15.40:0/1923721708 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f3d4400b200 con 0x7f3d54071cf0 2023-11-17T05:21:26.416 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:15.176+0000 7f3d5a3df700 1 --2- 172.21.15.40:0/1923721708 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f3d54071cf0 0x7f3d54110fb0 secure :-1 s=READY pgs=86 cs=0 l=1 rev1=1 rx=0x7f3d44011820 tx=0x7f3d44006ee0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:21:26.416 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:15.176+0000 7f3d42ffd700 1 -- 172.21.15.40:0/1923721708 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f3d4401b410 con 0x7f3d54071cf0 2023-11-17T05:21:26.417 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:15.176+0000 7f3d42ffd700 1 -- 172.21.15.40:0/1923721708 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(25 keys) v1 ==== 1010+0+0 (secure 0 0 0) 0x7f3d4401b9e0 con 0x7f3d54071cf0 2023-11-17T05:21:26.417 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:15.176+0000 7f3d42ffd700 1 -- 172.21.15.40:0/1923721708 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f3d4401a8b0 con 0x7f3d54071cf0 2023-11-17T05:21:26.417 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:15.176+0000 7f3d5c643700 1 -- 172.21.15.40:0/1923721708 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f3d541116a0 con 0x7f3d54071cf0 2023-11-17T05:21:26.417 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:15.176+0000 7f3d5c643700 1 -- 172.21.15.40:0/1923721708 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f3d54111ab0 con 0x7f3d54071cf0 2023-11-17T05:21:26.417 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:15.177+0000 7f3d42ffd700 1 -- 172.21.15.40:0/1923721708 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 16) v1 ==== 36208+0+0 (secure 0 0 0) 0x7f3d4400f070 con 0x7f3d54071cf0 2023-11-17T05:21:26.417 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:15.178+0000 7f3d5c643700 1 -- 172.21.15.40:0/1923721708 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f3d54045ad0 con 0x7f3d54071cf0 2023-11-17T05:21:26.417 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:15.178+0000 7f3d42ffd700 1 --2- 172.21.15.40:0/1923721708 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f3d3c02ff00 0x7f3d3c0323c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:21:26.417 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:15.178+0000 7f3d42ffd700 1 -- 172.21.15.40:0/1923721708 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(4..4 src has 1..4) v4 ==== 1270+0+0 (secure 0 0 0) 0x7f3d44041570 con 0x7f3d54071cf0 2023-11-17T05:21:26.417 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:15.178+0000 7f3d59bde700 1 --2- 172.21.15.40:0/1923721708 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f3d3c02ff00 0x7f3d3c0323c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:21:26.417 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:15.178+0000 7f3d59bde700 1 --2- 172.21.15.40:0/1923721708 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f3d3c02ff00 0x7f3d3c0323c0 secure :-1 s=READY pgs=8 cs=0 l=1 rev1=1 rx=0x7f3d500096a0 tx=0x7f3d50006b40).ready entity=mgr.14164 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:21:26.417 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:15.182+0000 7f3d42ffd700 1 -- 172.21.15.40:0/1923721708 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+152212 (secure 0 0 0) 0x7f3d44019cd0 con 0x7f3d54071cf0 2023-11-17T05:21:26.417 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:15.346+0000 7f3d5c643700 1 -- 172.21.15.40:0/1923721708 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "config get", "who": "mgr", "key": "mgr/dashboard/ssl_server_port"} v 0) v1 -- 0x7f3d541129f0 con 0x7f3d54071cf0 2023-11-17T05:21:26.417 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:15.347+0000 7f3d42ffd700 1 -- 172.21.15.40:0/1923721708 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mon_command_ack([{"prefix": "config get", "who": "mgr", "key": "mgr/dashboard/ssl_server_port"}]=0 v9) v1 ==== 112+0+5 (secure 0 0 0) 0x7f3d44016070 con 0x7f3d54071cf0 2023-11-17T05:21:26.418 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 8443 2023-11-17T05:21:26.418 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:15.348+0000 7f3d5c643700 1 -- 172.21.15.40:0/1923721708 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f3d3c02ff00 msgr2=0x7f3d3c0323c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:21:26.418 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:15.348+0000 7f3d5c643700 1 --2- 172.21.15.40:0/1923721708 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f3d3c02ff00 0x7f3d3c0323c0 secure :-1 s=READY pgs=8 cs=0 l=1 rev1=1 rx=0x7f3d500096a0 tx=0x7f3d50006b40).stop 2023-11-17T05:21:26.418 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:15.348+0000 7f3d5c643700 1 -- 172.21.15.40:0/1923721708 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f3d54071cf0 msgr2=0x7f3d54110fb0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:21:26.418 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:15.348+0000 7f3d5c643700 1 --2- 172.21.15.40:0/1923721708 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f3d54071cf0 0x7f3d54110fb0 secure :-1 s=READY pgs=86 cs=0 l=1 rev1=1 rx=0x7f3d44011820 tx=0x7f3d44006ee0).stop 2023-11-17T05:21:26.418 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:15.349+0000 7f3d5c643700 1 -- 172.21.15.40:0/1923721708 shutdown_connections 2023-11-17T05:21:26.418 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:15.349+0000 7f3d5c643700 1 --2- 172.21.15.40:0/1923721708 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f3d3c02ff00 0x7f3d3c0323c0 unknown :-1 s=CLOSED pgs=8 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:21:26.418 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:15.349+0000 7f3d5c643700 1 --2- 172.21.15.40:0/1923721708 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f3d54071cf0 0x7f3d54110fb0 unknown :-1 s=CLOSED pgs=86 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:21:26.418 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:15.349+0000 7f3d5c643700 1 -- 172.21.15.40:0/1923721708 >> 172.21.15.40:0/1923721708 conn(0x7f3d540f73f0 msgr2=0x7f3d540fcfa0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:21:26.418 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:15.349+0000 7f3d5c643700 1 -- 172.21.15.40:0/1923721708 shutdown_connections 2023-11-17T05:21:26.418 INFO:teuthology.orchestra.run.smithi040.stderr:/usr/bin/ceph: 2023-11-17T05:21:15.349+0000 7f3d5c643700 1 -- 172.21.15.40:0/1923721708 wait complete. 2023-11-17T05:21:26.418 INFO:teuthology.orchestra.run.smithi040.stderr:Running command: systemctl is-enabled firewalld.service 2023-11-17T05:21:26.419 INFO:teuthology.orchestra.run.smithi040.stderr:systemctl: disabled 2023-11-17T05:21:26.419 INFO:teuthology.orchestra.run.smithi040.stderr:Running command: systemctl is-active firewalld.service 2023-11-17T05:21:26.419 INFO:teuthology.orchestra.run.smithi040.stderr:systemctl: inactive 2023-11-17T05:21:26.419 INFO:teuthology.orchestra.run.smithi040.stderr:firewalld.service is not enabled 2023-11-17T05:21:26.419 INFO:teuthology.orchestra.run.smithi040.stderr:Not possible to open ports <[8443]>. firewalld.service is not available 2023-11-17T05:21:26.419 INFO:teuthology.orchestra.run.smithi040.stderr:Ceph Dashboard is now available at: 2023-11-17T05:21:26.419 INFO:teuthology.orchestra.run.smithi040.stderr: 2023-11-17T05:21:26.419 INFO:teuthology.orchestra.run.smithi040.stderr: URL: https://smithi040.front.sepia.ceph.com:8443/ 2023-11-17T05:21:26.419 INFO:teuthology.orchestra.run.smithi040.stderr: User: admin 2023-11-17T05:21:26.419 INFO:teuthology.orchestra.run.smithi040.stderr: Password: 4ln25nhjwe 2023-11-17T05:21:26.419 INFO:teuthology.orchestra.run.smithi040.stderr: 2023-11-17T05:21:26.419 INFO:teuthology.orchestra.run.smithi040.stderr:You can access the Ceph CLI with: 2023-11-17T05:21:26.419 INFO:teuthology.orchestra.run.smithi040.stderr: 2023-11-17T05:21:26.420 INFO:teuthology.orchestra.run.smithi040.stderr: sudo /home/ubuntu/cephtest/cephadm shell --fsid ed6470e2-8508-11ee-8dbe-212e2dc638e7 -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring 2023-11-17T05:21:26.420 INFO:teuthology.orchestra.run.smithi040.stderr: 2023-11-17T05:21:26.420 INFO:teuthology.orchestra.run.smithi040.stderr:Please consider enabling telemetry to help improve Ceph: 2023-11-17T05:21:26.420 INFO:teuthology.orchestra.run.smithi040.stderr: 2023-11-17T05:21:26.420 INFO:teuthology.orchestra.run.smithi040.stderr: ceph telemetry on 2023-11-17T05:21:26.420 INFO:teuthology.orchestra.run.smithi040.stderr: 2023-11-17T05:21:26.420 INFO:teuthology.orchestra.run.smithi040.stderr:For more information see: 2023-11-17T05:21:26.420 INFO:teuthology.orchestra.run.smithi040.stderr: 2023-11-17T05:21:26.420 INFO:teuthology.orchestra.run.smithi040.stderr: https://docs.ceph.com/docs/pacific/mgr/telemetry/ 2023-11-17T05:21:26.420 INFO:teuthology.orchestra.run.smithi040.stderr: 2023-11-17T05:21:26.420 INFO:teuthology.orchestra.run.smithi040.stderr:Bootstrap complete. 2023-11-17T05:21:26.420 INFO:teuthology.orchestra.run.smithi040.stderr:Releasing lock 139692835310056 on /run/cephadm/ed6470e2-8508-11ee-8dbe-212e2dc638e7.lock 2023-11-17T05:21:26.420 INFO:teuthology.orchestra.run.smithi040.stderr:Lock 139692835310056 released on /run/cephadm/ed6470e2-8508-11ee-8dbe-212e2dc638e7.lock 2023-11-17T05:21:26.421 INFO:tasks.cephadm:Fetching config... 2023-11-17T05:21:26.421 DEBUG:teuthology.orchestra.run.smithi040:> set -ex 2023-11-17T05:21:26.422 DEBUG:teuthology.orchestra.run.smithi040:> dd if=/etc/ceph/ceph.conf of=/dev/stdout 2023-11-17T05:21:26.447 INFO:tasks.cephadm:Fetching client.admin keyring... 2023-11-17T05:21:26.447 DEBUG:teuthology.orchestra.run.smithi040:> set -ex 2023-11-17T05:21:26.447 DEBUG:teuthology.orchestra.run.smithi040:> dd if=/etc/ceph/ceph.client.admin.keyring of=/dev/stdout 2023-11-17T05:21:26.511 INFO:tasks.cephadm:Fetching mon keyring... 2023-11-17T05:21:26.512 DEBUG:teuthology.orchestra.run.smithi040:> set -ex 2023-11-17T05:21:26.512 DEBUG:teuthology.orchestra.run.smithi040:> sudo dd if=/var/lib/ceph/ed6470e2-8508-11ee-8dbe-212e2dc638e7/mon.smithi040/keyring of=/dev/stdout 2023-11-17T05:21:26.591 INFO:tasks.cephadm:Fetching pub ssh key... 2023-11-17T05:21:26.591 DEBUG:teuthology.orchestra.run.smithi040:> set -ex 2023-11-17T05:21:26.591 DEBUG:teuthology.orchestra.run.smithi040:> dd if=/home/ubuntu/cephtest/ceph.pub of=/dev/stdout 2023-11-17T05:21:26.654 INFO:tasks.cephadm:Installing pub ssh key for root users... 2023-11-17T05:21:26.655 DEBUG:teuthology.orchestra.run.smithi040:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDL+SdWMF1qjsHjggKNNpIDFxDY6oTIkPdEqTXFMkmwahWZQUyjp7g+1PyiRV6tvkqmLIRshV75Qr9NMWGqY8e9+o0VBppbADFtyq254i71kMbLUcWa9YzDpgOslDiF5PpfCba9Bhh2XoIwwmPScUUP653EzFNllt2VVFiPZOT7UShuA0viW2dm+gQSY+L8IdDfSFD75vgPm8V2WQdsxKwIoGB0LRD0m5MAoP5+XrMxgLiymH5i4d2k2KfyNAsXKcEZeSFCrun43rXcziCK3kiCIMWuiUIEme2Kuu46MLaRZzuw2MYz3Fx4p/5akiQYxl2JDuBpDHCCxOw2ZOoAT7Hzr1jOpViifPKQ1lFCDpPsxjci/kDbAKhetG/T9e2D9R3nAMmqi/H3fVMSoOIneIS+cWOtK+ZIpFwxcmfVI7NlHNNV4Ckg0xaqAfx5kf/PwK+8vzfkbutSNAIGJ1o6i4Y+70usB7wkLU9r26fs/UsEH7Lip4rTSdQhp+ToebHpBsc= ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2023-11-17T05:21:26.742 INFO:teuthology.orchestra.run.smithi040.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDL+SdWMF1qjsHjggKNNpIDFxDY6oTIkPdEqTXFMkmwahWZQUyjp7g+1PyiRV6tvkqmLIRshV75Qr9NMWGqY8e9+o0VBppbADFtyq254i71kMbLUcWa9YzDpgOslDiF5PpfCba9Bhh2XoIwwmPScUUP653EzFNllt2VVFiPZOT7UShuA0viW2dm+gQSY+L8IdDfSFD75vgPm8V2WQdsxKwIoGB0LRD0m5MAoP5+XrMxgLiymH5i4d2k2KfyNAsXKcEZeSFCrun43rXcziCK3kiCIMWuiUIEme2Kuu46MLaRZzuw2MYz3Fx4p/5akiQYxl2JDuBpDHCCxOw2ZOoAT7Hzr1jOpViifPKQ1lFCDpPsxjci/kDbAKhetG/T9e2D9R3nAMmqi/H3fVMSoOIneIS+cWOtK+ZIpFwxcmfVI7NlHNNV4Ckg0xaqAfx5kf/PwK+8vzfkbutSNAIGJ1o6i4Y+70usB7wkLU9r26fs/UsEH7Lip4rTSdQhp+ToebHpBsc= ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7 2023-11-17T05:21:26.757 DEBUG:teuthology.orchestra.run.smithi066:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDL+SdWMF1qjsHjggKNNpIDFxDY6oTIkPdEqTXFMkmwahWZQUyjp7g+1PyiRV6tvkqmLIRshV75Qr9NMWGqY8e9+o0VBppbADFtyq254i71kMbLUcWa9YzDpgOslDiF5PpfCba9Bhh2XoIwwmPScUUP653EzFNllt2VVFiPZOT7UShuA0viW2dm+gQSY+L8IdDfSFD75vgPm8V2WQdsxKwIoGB0LRD0m5MAoP5+XrMxgLiymH5i4d2k2KfyNAsXKcEZeSFCrun43rXcziCK3kiCIMWuiUIEme2Kuu46MLaRZzuw2MYz3Fx4p/5akiQYxl2JDuBpDHCCxOw2ZOoAT7Hzr1jOpViifPKQ1lFCDpPsxjci/kDbAKhetG/T9e2D9R3nAMmqi/H3fVMSoOIneIS+cWOtK+ZIpFwxcmfVI7NlHNNV4Ckg0xaqAfx5kf/PwK+8vzfkbutSNAIGJ1o6i4Y+70usB7wkLU9r26fs/UsEH7Lip4rTSdQhp+ToebHpBsc= ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2023-11-17T05:21:26.803 INFO:teuthology.orchestra.run.smithi066.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDL+SdWMF1qjsHjggKNNpIDFxDY6oTIkPdEqTXFMkmwahWZQUyjp7g+1PyiRV6tvkqmLIRshV75Qr9NMWGqY8e9+o0VBppbADFtyq254i71kMbLUcWa9YzDpgOslDiF5PpfCba9Bhh2XoIwwmPScUUP653EzFNllt2VVFiPZOT7UShuA0viW2dm+gQSY+L8IdDfSFD75vgPm8V2WQdsxKwIoGB0LRD0m5MAoP5+XrMxgLiymH5i4d2k2KfyNAsXKcEZeSFCrun43rXcziCK3kiCIMWuiUIEme2Kuu46MLaRZzuw2MYz3Fx4p/5akiQYxl2JDuBpDHCCxOw2ZOoAT7Hzr1jOpViifPKQ1lFCDpPsxjci/kDbAKhetG/T9e2D9R3nAMmqi/H3fVMSoOIneIS+cWOtK+ZIpFwxcmfVI7NlHNNV4Ckg0xaqAfx5kf/PwK+8vzfkbutSNAIGJ1o6i4Y+70usB7wkLU9r26fs/UsEH7Lip4rTSdQhp+ToebHpBsc= ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7 2023-11-17T05:21:26.816 DEBUG:teuthology.orchestra.run.smithi040:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.4 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ed6470e2-8508-11ee-8dbe-212e2dc638e7 -- ceph config set mgr mgr/cephadm/allow_ptrace true 2023-11-17T05:21:27.986 INFO:teuthology.orchestra.run.smithi040.stderr:2023-11-17T05:21:27.983+0000 7f25ec626700 1 -- 172.21.15.40:0/748611246 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f25e40f5410 msgr2=0x7f25e40f5830 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:21:27.987 INFO:teuthology.orchestra.run.smithi040.stderr:2023-11-17T05:21:27.983+0000 7f25ec626700 1 --2- 172.21.15.40:0/748611246 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f25e40f5410 0x7f25e40f5830 secure :-1 s=READY pgs=87 cs=0 l=1 rev1=1 rx=0x7f25e0009fb0 tx=0x7f25e0009b30).stop 2023-11-17T05:21:27.987 INFO:teuthology.orchestra.run.smithi040.stderr:2023-11-17T05:21:27.983+0000 7f25ec626700 1 -- 172.21.15.40:0/748611246 shutdown_connections 2023-11-17T05:21:27.987 INFO:teuthology.orchestra.run.smithi040.stderr:2023-11-17T05:21:27.983+0000 7f25ec626700 1 --2- 172.21.15.40:0/748611246 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f25e40f5410 0x7f25e40f5830 unknown :-1 s=CLOSED pgs=87 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:21:27.987 INFO:teuthology.orchestra.run.smithi040.stderr:2023-11-17T05:21:27.983+0000 7f25ec626700 1 -- 172.21.15.40:0/748611246 >> 172.21.15.40:0/748611246 conn(0x7f25e40f0b70 msgr2=0x7f25e40f2f90 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:21:27.987 INFO:teuthology.orchestra.run.smithi040.stderr:2023-11-17T05:21:27.983+0000 7f25ec626700 1 -- 172.21.15.40:0/748611246 shutdown_connections 2023-11-17T05:21:27.987 INFO:teuthology.orchestra.run.smithi040.stderr:2023-11-17T05:21:27.983+0000 7f25ec626700 1 -- 172.21.15.40:0/748611246 wait complete. 2023-11-17T05:21:27.987 INFO:teuthology.orchestra.run.smithi040.stderr:2023-11-17T05:21:27.984+0000 7f25ec626700 1 Processor -- start 2023-11-17T05:21:27.987 INFO:teuthology.orchestra.run.smithi040.stderr:2023-11-17T05:21:27.984+0000 7f25ec626700 1 -- start start 2023-11-17T05:21:27.987 INFO:teuthology.orchestra.run.smithi040.stderr:2023-11-17T05:21:27.984+0000 7f25ec626700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f25e40f5410 0x7f25e4106720 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:21:27.987 INFO:teuthology.orchestra.run.smithi040.stderr:2023-11-17T05:21:27.984+0000 7f25ec626700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f25e4106c10 con 0x7f25e40f5410 2023-11-17T05:21:27.987 INFO:teuthology.orchestra.run.smithi040.stderr:2023-11-17T05:21:27.984+0000 7f25ea3c2700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f25e40f5410 0x7f25e4106720 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:21:27.987 INFO:teuthology.orchestra.run.smithi040.stderr:2023-11-17T05:21:27.984+0000 7f25ea3c2700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f25e40f5410 0x7f25e4106720 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:55080/0 (socket says 172.21.15.40:55080) 2023-11-17T05:21:27.988 INFO:teuthology.orchestra.run.smithi040.stderr:2023-11-17T05:21:27.984+0000 7f25ea3c2700 1 -- 172.21.15.40:0/3051567258 learned_addr learned my addr 172.21.15.40:0/3051567258 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-11-17T05:21:27.988 INFO:teuthology.orchestra.run.smithi040.stderr:2023-11-17T05:21:27.984+0000 7f25ea3c2700 1 -- 172.21.15.40:0/3051567258 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f25e0005040 con 0x7f25e40f5410 2023-11-17T05:21:27.988 INFO:teuthology.orchestra.run.smithi040.stderr:2023-11-17T05:21:27.985+0000 7f25ea3c2700 1 --2- 172.21.15.40:0/3051567258 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f25e40f5410 0x7f25e4106720 secure :-1 s=READY pgs=88 cs=0 l=1 rev1=1 rx=0x7f25e0013510 tx=0x7f25e000b040).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:21:27.988 INFO:teuthology.orchestra.run.smithi040.stderr:2023-11-17T05:21:27.985+0000 7f25daffd700 1 -- 172.21.15.40:0/3051567258 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f25e0009320 con 0x7f25e40f5410 2023-11-17T05:21:27.988 INFO:teuthology.orchestra.run.smithi040.stderr:2023-11-17T05:21:27.985+0000 7f25daffd700 1 -- 172.21.15.40:0/3051567258 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(25 keys) v1 ==== 1074+0+0 (secure 0 0 0) 0x7f25e0013f00 con 0x7f25e40f5410 2023-11-17T05:21:27.988 INFO:teuthology.orchestra.run.smithi040.stderr:2023-11-17T05:21:27.985+0000 7f25ec626700 1 -- 172.21.15.40:0/3051567258 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f25e4106e10 con 0x7f25e40f5410 2023-11-17T05:21:27.988 INFO:teuthology.orchestra.run.smithi040.stderr:2023-11-17T05:21:27.985+0000 7f25daffd700 1 -- 172.21.15.40:0/3051567258 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f25e0014d90 con 0x7f25e40f5410 2023-11-17T05:21:27.988 INFO:teuthology.orchestra.run.smithi040.stderr:2023-11-17T05:21:27.985+0000 7f25ec626700 1 -- 172.21.15.40:0/3051567258 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f25e41072a0 con 0x7f25e40f5410 2023-11-17T05:21:27.989 INFO:teuthology.orchestra.run.smithi040.stderr:2023-11-17T05:21:27.986+0000 7f25daffd700 1 -- 172.21.15.40:0/3051567258 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 17) v1 ==== 36269+0+0 (secure 0 0 0) 0x7f25e00263a0 con 0x7f25e40f5410 2023-11-17T05:21:27.989 INFO:teuthology.orchestra.run.smithi040.stderr:2023-11-17T05:21:27.986+0000 7f25ec626700 1 -- 172.21.15.40:0/3051567258 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f25e4045ad0 con 0x7f25e40f5410 2023-11-17T05:21:27.989 INFO:teuthology.orchestra.run.smithi040.stderr:2023-11-17T05:21:27.986+0000 7f25daffd700 1 --2- 172.21.15.40:0/3051567258 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f25d002ffb0 0x7f25d0032470 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:21:27.989 INFO:teuthology.orchestra.run.smithi040.stderr:2023-11-17T05:21:27.986+0000 7f25daffd700 1 -- 172.21.15.40:0/3051567258 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(4..4 src has 1..4) v4 ==== 1270+0+0 (secure 0 0 0) 0x7f25e0040fb0 con 0x7f25e40f5410 2023-11-17T05:21:27.989 INFO:teuthology.orchestra.run.smithi040.stderr:2023-11-17T05:21:27.986+0000 7f25e9bc1700 1 --2- 172.21.15.40:0/3051567258 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f25d002ffb0 0x7f25d0032470 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:21:27.990 INFO:teuthology.orchestra.run.smithi040.stderr:2023-11-17T05:21:27.987+0000 7f25e9bc1700 1 --2- 172.21.15.40:0/3051567258 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f25d002ffb0 0x7f25d0032470 secure :-1 s=READY pgs=9 cs=0 l=1 rev1=1 rx=0x7f25d40096a0 tx=0x7f25d4006b40).ready entity=mgr.14164 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:21:27.992 INFO:teuthology.orchestra.run.smithi040.stderr:2023-11-17T05:21:27.991+0000 7f25daffd700 1 -- 172.21.15.40:0/3051567258 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+152212 (secure 0 0 0) 0x7f25e006d2b0 con 0x7f25e40f5410 2023-11-17T05:21:28.159 INFO:teuthology.orchestra.run.smithi040.stderr:2023-11-17T05:21:28.157+0000 7f25ec626700 1 -- 172.21.15.40:0/3051567258 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command([{prefix=config set, name=mgr/cephadm/allow_ptrace}] v 0) v1 -- 0x7f25e4108160 con 0x7f25e40f5410 2023-11-17T05:21:28.162 INFO:teuthology.orchestra.run.smithi040.stderr:2023-11-17T05:21:28.161+0000 7f25daffd700 1 -- 172.21.15.40:0/3051567258 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mon_command_ack([{prefix=config set, name=mgr/cephadm/allow_ptrace}]=0 v11)=0 v11) v1 ==== 125+0+0 (secure 0 0 0) 0x7f25e00126f0 con 0x7f25e40f5410 2023-11-17T05:21:28.164 INFO:teuthology.orchestra.run.smithi040.stderr:2023-11-17T05:21:28.163+0000 7f25ec626700 1 -- 172.21.15.40:0/3051567258 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f25d002ffb0 msgr2=0x7f25d0032470 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:21:28.165 INFO:teuthology.orchestra.run.smithi040.stderr:2023-11-17T05:21:28.163+0000 7f25ec626700 1 --2- 172.21.15.40:0/3051567258 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f25d002ffb0 0x7f25d0032470 secure :-1 s=READY pgs=9 cs=0 l=1 rev1=1 rx=0x7f25d40096a0 tx=0x7f25d4006b40).stop 2023-11-17T05:21:28.165 INFO:teuthology.orchestra.run.smithi040.stderr:2023-11-17T05:21:28.163+0000 7f25ec626700 1 -- 172.21.15.40:0/3051567258 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f25e40f5410 msgr2=0x7f25e4106720 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:21:28.165 INFO:teuthology.orchestra.run.smithi040.stderr:2023-11-17T05:21:28.163+0000 7f25ec626700 1 --2- 172.21.15.40:0/3051567258 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f25e40f5410 0x7f25e4106720 secure :-1 s=READY pgs=88 cs=0 l=1 rev1=1 rx=0x7f25e0013510 tx=0x7f25e000b040).stop 2023-11-17T05:21:28.165 INFO:teuthology.orchestra.run.smithi040.stderr:2023-11-17T05:21:28.163+0000 7f25ec626700 1 -- 172.21.15.40:0/3051567258 shutdown_connections 2023-11-17T05:21:28.165 INFO:teuthology.orchestra.run.smithi040.stderr:2023-11-17T05:21:28.163+0000 7f25ec626700 1 --2- 172.21.15.40:0/3051567258 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f25d002ffb0 0x7f25d0032470 unknown :-1 s=CLOSED pgs=9 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:21:28.165 INFO:teuthology.orchestra.run.smithi040.stderr:2023-11-17T05:21:28.163+0000 7f25ec626700 1 --2- 172.21.15.40:0/3051567258 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f25e40f5410 0x7f25e4106720 unknown :-1 s=CLOSED pgs=88 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:21:28.165 INFO:teuthology.orchestra.run.smithi040.stderr:2023-11-17T05:21:28.163+0000 7f25ec626700 1 -- 172.21.15.40:0/3051567258 >> 172.21.15.40:0/3051567258 conn(0x7f25e40f0b70 msgr2=0x7f25e405fe20 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:21:28.165 INFO:teuthology.orchestra.run.smithi040.stderr:2023-11-17T05:21:28.163+0000 7f25ec626700 1 -- 172.21.15.40:0/3051567258 shutdown_connections 2023-11-17T05:21:28.165 INFO:teuthology.orchestra.run.smithi040.stderr:2023-11-17T05:21:28.163+0000 7f25ec626700 1 -- 172.21.15.40:0/3051567258 wait complete. 2023-11-17T05:21:28.552 INFO:tasks.cephadm:Writing (initial) conf and keyring to smithi066 2023-11-17T05:21:28.552 DEBUG:teuthology.orchestra.run.smithi066:> set -ex 2023-11-17T05:21:28.553 DEBUG:teuthology.orchestra.run.smithi066:> dd of=/etc/ceph/ceph.conf 2023-11-17T05:21:28.575 DEBUG:teuthology.orchestra.run.smithi066:> set -ex 2023-11-17T05:21:28.575 DEBUG:teuthology.orchestra.run.smithi066:> dd of=/etc/ceph/ceph.client.admin.keyring 2023-11-17T05:21:28.636 INFO:tasks.cephadm:Adding host smithi066 to orchestrator... 2023-11-17T05:21:28.636 DEBUG:teuthology.orchestra.run.smithi066:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.4 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ed6470e2-8508-11ee-8dbe-212e2dc638e7 -- ceph orch host add smithi066 2023-11-17T05:21:29.489 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:21:29 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:21:28.162571+0000 mon.smithi040 (mon.0) 116 : audit [INF] from='client.? 172.21.15.40:0/3051567258' entity='client.admin' 2023-11-17T05:21:32.489 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:21:32 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:21:31.157816+0000 mgr.smithi040.ogcprl (mgr.14164) 6 : cluster [DBG] pgmap v3: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:21:34.489 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:21:34 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:21:33.158178+0000 mgr.smithi040.ogcprl (mgr.14164) 7 : cluster [DBG] pgmap v4: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:21:36.739 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:21:36 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:21:35.158516+0000 mgr.smithi040.ogcprl (mgr.14164) 8 : cluster [DBG] pgmap v5: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:21:38.739 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:21:38 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:21:37.158867+0000 mgr.smithi040.ogcprl (mgr.14164) 9 : cluster [DBG] pgmap v6: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:21:40.739 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:21:40 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:21:39.159240+0000 mgr.smithi040.ogcprl (mgr.14164) 10 : cluster [DBG] pgmap v7: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:21:42.739 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:21:42 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:21:41.159489+0000 mgr.smithi040.ogcprl (mgr.14164) 11 : cluster [DBG] pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:21:44.739 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:21:44 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:21:43.159838+0000 mgr.smithi040.ogcprl (mgr.14164) 12 : cluster [DBG] pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:21:46.739 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:21:46 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:21:45.160193+0000 mgr.smithi040.ogcprl (mgr.14164) 13 : cluster [DBG] pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:21:48.739 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:21:48 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:21:47.160379+0000 mgr.smithi040.ogcprl (mgr.14164) 14 : cluster [DBG] pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:21:50.739 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:21:50 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:21:49.160638+0000 mgr.smithi040.ogcprl (mgr.14164) 15 : cluster [DBG] pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:21:52.739 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:21:52 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:21:51.160908+0000 mgr.smithi040.ogcprl (mgr.14164) 16 : cluster [DBG] pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:21:54.474 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:21:54.472+0000 7f03b7fff700 1 -- 172.21.15.66:0/2309659950 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f03b80f5810 msgr2=0x7f03b80f5c30 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:21:54.474 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:21:54.472+0000 7f03b7fff700 1 --2- 172.21.15.66:0/2309659950 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f03b80f5810 0x7f03b80f5c30 secure :-1 s=READY pgs=89 cs=0 l=1 rev1=1 rx=0x7f03a8009fb0 tx=0x7f03a8009b30).stop 2023-11-17T05:21:54.474 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:21:54.472+0000 7f03b7fff700 1 -- 172.21.15.66:0/2309659950 shutdown_connections 2023-11-17T05:21:54.474 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:21:54.472+0000 7f03b7fff700 1 --2- 172.21.15.66:0/2309659950 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f03b80f5810 0x7f03b80f5c30 unknown :-1 s=CLOSED pgs=89 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:21:54.475 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:21:54.472+0000 7f03b7fff700 1 -- 172.21.15.66:0/2309659950 >> 172.21.15.66:0/2309659950 conn(0x7f03b80f0120 msgr2=0x7f03b80f2540 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:21:54.475 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:21:54.472+0000 7f03b7fff700 1 -- 172.21.15.66:0/2309659950 shutdown_connections 2023-11-17T05:21:54.475 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:21:54.472+0000 7f03b7fff700 1 -- 172.21.15.66:0/2309659950 wait complete. 2023-11-17T05:21:54.475 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:21:54.472+0000 7f03b7fff700 1 Processor -- start 2023-11-17T05:21:54.475 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:21:54.472+0000 7f03b7fff700 1 -- start start 2023-11-17T05:21:54.475 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:21:54.473+0000 7f03b7fff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f03b80f5810 0x7f03b8109200 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:21:54.475 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:21:54.473+0000 7f03b7fff700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f03b80f51d0 con 0x7f03b80f5810 2023-11-17T05:21:54.475 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:21:54.473+0000 7f03b6ffd700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f03b80f5810 0x7f03b8109200 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:21:54.475 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:21:54.473+0000 7f03b6ffd700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f03b80f5810 0x7f03b8109200 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.66:57970/0 (socket says 172.21.15.66:57970) 2023-11-17T05:21:54.475 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:21:54.473+0000 7f03b6ffd700 1 -- 172.21.15.66:0/2988174281 learned_addr learned my addr 172.21.15.66:0/2988174281 (peer_addr_for_me v2:172.21.15.66:0/0) 2023-11-17T05:21:54.475 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:21:54.473+0000 7f03b6ffd700 1 -- 172.21.15.66:0/2988174281 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f03a8005040 con 0x7f03b80f5810 2023-11-17T05:21:54.475 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:21:54.473+0000 7f03b6ffd700 1 --2- 172.21.15.66:0/2988174281 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f03b80f5810 0x7f03b8109200 secure :-1 s=READY pgs=90 cs=0 l=1 rev1=1 rx=0x7f03a80079e0 tx=0x7f03a8009b30).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:21:54.476 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:21:54.473+0000 7f039f7fe700 1 -- 172.21.15.66:0/2988174281 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f03a8018070 con 0x7f03b80f5810 2023-11-17T05:21:54.476 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:21:54.474+0000 7f03b7fff700 1 -- 172.21.15.66:0/2988174281 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f03b80f53d0 con 0x7f03b80f5810 2023-11-17T05:21:54.476 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:21:54.474+0000 7f039f7fe700 1 -- 172.21.15.66:0/2988174281 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(25 keys) v1 ==== 1074+0+0 (secure 0 0 0) 0x7f03a8013410 con 0x7f03b80f5810 2023-11-17T05:21:54.476 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:21:54.474+0000 7f039f7fe700 1 -- 172.21.15.66:0/2988174281 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f03a801c3c0 con 0x7f03b80f5810 2023-11-17T05:21:54.476 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:21:54.474+0000 7f03b7fff700 1 -- 172.21.15.66:0/2988174281 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f03b81098b0 con 0x7f03b80f5810 2023-11-17T05:21:54.477 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:21:54.474+0000 7f039f7fe700 1 -- 172.21.15.66:0/2988174281 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 17) v1 ==== 36269+0+0 (secure 0 0 0) 0x7f03a801d420 con 0x7f03b80f5810 2023-11-17T05:21:54.477 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:21:54.474+0000 7f03b7fff700 1 -- 172.21.15.66:0/2988174281 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f03b8059100 con 0x7f03b80f5810 2023-11-17T05:21:54.477 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:21:54.475+0000 7f039f7fe700 1 --2- 172.21.15.66:0/2988174281 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f03a002ff60 0x7f03a0032420 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:21:54.477 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:21:54.475+0000 7f039f7fe700 1 -- 172.21.15.66:0/2988174281 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(4..4 src has 1..4) v4 ==== 1270+0+0 (secure 0 0 0) 0x7f03a8041460 con 0x7f03b80f5810 2023-11-17T05:21:54.477 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:21:54.475+0000 7f03b67fc700 1 --2- 172.21.15.66:0/2988174281 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f03a002ff60 0x7f03a0032420 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:21:54.477 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:21:54.475+0000 7f03b67fc700 1 --2- 172.21.15.66:0/2988174281 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f03a002ff60 0x7f03a0032420 secure :-1 s=READY pgs=10 cs=0 l=1 rev1=1 rx=0x7f03ac0096a0 tx=0x7f03ac006b40).ready entity=mgr.14164 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:21:54.479 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:21:54.479+0000 7f039f7fe700 1 -- 172.21.15.66:0/2988174281 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+152212 (secure 0 0 0) 0x7f03a801d7e0 con 0x7f03b80f5810 2023-11-17T05:21:54.662 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:21:54.662+0000 7f03b7fff700 1 -- 172.21.15.66:0/2988174281 --> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] -- mgr_command(tid 0: {"prefix": "orch host add", "hostname": "smithi066", "target": ["mon-mgr", ""]}) v1 -- 0x7f03b80f1300 con 0x7f03a002ff60 2023-11-17T05:21:54.698 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:21:54 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:21:53.161190+0000 mgr.smithi040.ogcprl (mgr.14164) 17 : cluster [DBG] pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:21:55.280 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:21:55.279+0000 7f039f7fe700 1 -- 172.21.15.66:0/2988174281 <== mgr.14164 v2:172.21.15.40:6800/1119389001 1 ==== mgr_command_reply(tid 0: 0 ) v1 ==== 8+0+23 (secure 0 0 0) 0x7f03b80f1300 con 0x7f03a002ff60 2023-11-17T05:21:55.281 INFO:teuthology.orchestra.run.smithi066.stdout:Added host 'smithi066' 2023-11-17T05:21:55.282 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:21:55.281+0000 7f03b7fff700 1 -- 172.21.15.66:0/2988174281 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f03a002ff60 msgr2=0x7f03a0032420 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:21:55.282 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:21:55.281+0000 7f03b7fff700 1 --2- 172.21.15.66:0/2988174281 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f03a002ff60 0x7f03a0032420 secure :-1 s=READY pgs=10 cs=0 l=1 rev1=1 rx=0x7f03ac0096a0 tx=0x7f03ac006b40).stop 2023-11-17T05:21:55.282 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:21:55.281+0000 7f03b7fff700 1 -- 172.21.15.66:0/2988174281 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f03b80f5810 msgr2=0x7f03b8109200 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:21:55.282 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:21:55.281+0000 7f03b7fff700 1 --2- 172.21.15.66:0/2988174281 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f03b80f5810 0x7f03b8109200 secure :-1 s=READY pgs=90 cs=0 l=1 rev1=1 rx=0x7f03a80079e0 tx=0x7f03a8009b30).stop 2023-11-17T05:21:55.282 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:21:55.281+0000 7f03b7fff700 1 -- 172.21.15.66:0/2988174281 shutdown_connections 2023-11-17T05:21:55.283 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:21:55.281+0000 7f03b7fff700 1 --2- 172.21.15.66:0/2988174281 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f03a002ff60 0x7f03a0032420 unknown :-1 s=CLOSED pgs=10 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:21:55.283 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:21:55.281+0000 7f03b7fff700 1 --2- 172.21.15.66:0/2988174281 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f03b80f5810 0x7f03b8109200 unknown :-1 s=CLOSED pgs=90 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:21:55.283 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:21:55.281+0000 7f03b7fff700 1 -- 172.21.15.66:0/2988174281 >> 172.21.15.66:0/2988174281 conn(0x7f03b80f0120 msgr2=0x7f03b80f0bf0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:21:55.283 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:21:55.282+0000 7f03b7fff700 1 -- 172.21.15.66:0/2988174281 shutdown_connections 2023-11-17T05:21:55.283 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:21:55.282+0000 7f03b7fff700 1 -- 172.21.15.66:0/2988174281 wait complete. 2023-11-17T05:21:55.618 DEBUG:teuthology.orchestra.run.smithi066:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.4 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ed6470e2-8508-11ee-8dbe-212e2dc638e7 -- ceph orch host ls --format=json 2023-11-17T05:21:56.739 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:21:56 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:21:54.663825+0000 mgr.smithi040.ogcprl (mgr.14164) 18 : audit [DBG] from='client.14182 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi066", "target": ["mon-mgr", ""]}]: dispatch 2023-11-17T05:21:56.739 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:21:56 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:21:55.161636+0000 mgr.smithi040.ogcprl (mgr.14164) 19 : cluster [DBG] pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:21:56.739 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:21:56 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:21:55.279689+0000 mon.smithi040 (mon.0) 117 : audit [INF] from='mgr.14164 172.21.15.40:0/2203032451' entity='mgr.smithi040.ogcprl' 2023-11-17T05:21:56.739 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:21:56 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cephadm 2023-11-17T05:21:55.280108+0000 mgr.smithi040.ogcprl (mgr.14164) 20 : cephadm [INF] Added host smithi066 2023-11-17T05:21:56.766 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:21:56.762+0000 7f5902c86700 1 -- 172.21.15.66:0/1599313675 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f58fc0f5410 msgr2=0x7f58fc0f5830 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:21:56.766 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:21:56.762+0000 7f5902c86700 1 --2- 172.21.15.66:0/1599313675 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f58fc0f5410 0x7f58fc0f5830 secure :-1 s=READY pgs=91 cs=0 l=1 rev1=1 rx=0x7f58f0009fb0 tx=0x7f58f0009b30).stop 2023-11-17T05:21:56.766 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:21:56.762+0000 7f5902c86700 1 -- 172.21.15.66:0/1599313675 shutdown_connections 2023-11-17T05:21:56.766 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:21:56.762+0000 7f5902c86700 1 --2- 172.21.15.66:0/1599313675 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f58fc0f5410 0x7f58fc0f5830 unknown :-1 s=CLOSED pgs=91 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:21:56.766 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:21:56.762+0000 7f5902c86700 1 -- 172.21.15.66:0/1599313675 >> 172.21.15.66:0/1599313675 conn(0x7f58fc0f0b70 msgr2=0x7f58fc0f2f90 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:21:56.767 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:21:56.762+0000 7f5902c86700 1 -- 172.21.15.66:0/1599313675 shutdown_connections 2023-11-17T05:21:56.767 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:21:56.762+0000 7f5902c86700 1 -- 172.21.15.66:0/1599313675 wait complete. 2023-11-17T05:21:56.767 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:21:56.763+0000 7f5902c86700 1 Processor -- start 2023-11-17T05:21:56.767 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:21:56.763+0000 7f5902c86700 1 -- start start 2023-11-17T05:21:56.767 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:21:56.763+0000 7f5902c86700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f58fc0f5410 0x7f58fc1093f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:21:56.767 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:21:56.763+0000 7f5902c86700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f58fc060280 con 0x7f58fc0f5410 2023-11-17T05:21:56.767 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:21:56.763+0000 7f5900a22700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f58fc0f5410 0x7f58fc1093f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:21:56.767 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:21:56.764+0000 7f5900a22700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f58fc0f5410 0x7f58fc1093f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.66:57982/0 (socket says 172.21.15.66:57982) 2023-11-17T05:21:56.767 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:21:56.764+0000 7f5900a22700 1 -- 172.21.15.66:0/1989670496 learned_addr learned my addr 172.21.15.66:0/1989670496 (peer_addr_for_me v2:172.21.15.66:0/0) 2023-11-17T05:21:56.767 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:21:56.764+0000 7f5900a22700 1 -- 172.21.15.66:0/1989670496 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f58f0005040 con 0x7f58fc0f5410 2023-11-17T05:21:56.768 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:21:56.764+0000 7f5900a22700 1 --2- 172.21.15.66:0/1989670496 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f58fc0f5410 0x7f58fc1093f0 secure :-1 s=READY pgs=92 cs=0 l=1 rev1=1 rx=0x7f58f0013510 tx=0x7f58f000b040).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:21:56.768 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:21:56.764+0000 7f58f97fa700 1 -- 172.21.15.66:0/1989670496 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f58f0009320 con 0x7f58fc0f5410 2023-11-17T05:21:56.768 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:21:56.764+0000 7f5902c86700 1 -- 172.21.15.66:0/1989670496 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f58fc060480 con 0x7f58fc0f5410 2023-11-17T05:21:56.768 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:21:56.764+0000 7f58f97fa700 1 -- 172.21.15.66:0/1989670496 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(25 keys) v1 ==== 1074+0+0 (secure 0 0 0) 0x7f58f0013f00 con 0x7f58fc0f5410 2023-11-17T05:21:56.768 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:21:56.764+0000 7f58f97fa700 1 -- 172.21.15.66:0/1989670496 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f58f0014d90 con 0x7f58fc0f5410 2023-11-17T05:21:56.768 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:21:56.764+0000 7f5902c86700 1 -- 172.21.15.66:0/1989670496 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f58fc109b20 con 0x7f58fc0f5410 2023-11-17T05:21:56.768 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:21:56.765+0000 7f58f97fa700 1 -- 172.21.15.66:0/1989670496 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 17) v1 ==== 36269+0+0 (secure 0 0 0) 0x7f58f00095f0 con 0x7f58fc0f5410 2023-11-17T05:21:56.768 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:21:56.765+0000 7f5902c86700 1 -- 172.21.15.66:0/1989670496 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f58fc045ad0 con 0x7f58fc0f5410 2023-11-17T05:21:56.769 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:21:56.765+0000 7f58f97fa700 1 --2- 172.21.15.66:0/1989670496 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f58ec02ff60 0x7f58ec032420 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:21:56.769 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:21:56.765+0000 7f58f97fa700 1 -- 172.21.15.66:0/1989670496 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(4..4 src has 1..4) v4 ==== 1270+0+0 (secure 0 0 0) 0x7f58f0041000 con 0x7f58fc0f5410 2023-11-17T05:21:56.769 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:21:56.765+0000 7f58fbfff700 1 --2- 172.21.15.66:0/1989670496 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f58ec02ff60 0x7f58ec032420 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:21:56.769 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:21:56.766+0000 7f58fbfff700 1 --2- 172.21.15.66:0/1989670496 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f58ec02ff60 0x7f58ec032420 secure :-1 s=READY pgs=11 cs=0 l=1 rev1=1 rx=0x7f58e80096a0 tx=0x7f58e8006b40).ready entity=mgr.14164 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:21:56.770 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:21:56.770+0000 7f58f97fa700 1 -- 172.21.15.66:0/1989670496 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+152212 (secure 0 0 0) 0x7f58f0014420 con 0x7f58fc0f5410 2023-11-17T05:21:56.955 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:21:56.954+0000 7f5902c86700 1 -- 172.21.15.66:0/1989670496 --> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] -- mgr_command(tid 0: {"prefix": "orch host ls", "target": ["mon-mgr", ""], "format": "json"}) v1 -- 0x7f58fc0f1d70 con 0x7f58ec02ff60 2023-11-17T05:21:56.956 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:21:56.956+0000 7f58f97fa700 1 -- 172.21.15.66:0/1989670496 <== mgr.14164 v2:172.21.15.40:6800/1119389001 1 ==== mgr_command_reply(tid 0: 0 ) v1 ==== 8+0+156 (secure 0 0 0) 0x7f58fc0f1d70 con 0x7f58ec02ff60 2023-11-17T05:21:56.956 INFO:teuthology.orchestra.run.smithi066.stdout: 2023-11-17T05:21:56.956 INFO:teuthology.orchestra.run.smithi066.stdout:[{"addr": "172.21.15.40", "hostname": "smithi040", "labels": [], "status": ""}, {"addr": "smithi066", "hostname": "smithi066", "labels": [], "status": ""}] 2023-11-17T05:21:56.958 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:21:56.957+0000 7f5902c86700 1 -- 172.21.15.66:0/1989670496 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f58ec02ff60 msgr2=0x7f58ec032420 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:21:56.958 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:21:56.957+0000 7f5902c86700 1 --2- 172.21.15.66:0/1989670496 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f58ec02ff60 0x7f58ec032420 secure :-1 s=READY pgs=11 cs=0 l=1 rev1=1 rx=0x7f58e80096a0 tx=0x7f58e8006b40).stop 2023-11-17T05:21:56.958 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:21:56.957+0000 7f5902c86700 1 -- 172.21.15.66:0/1989670496 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f58fc0f5410 msgr2=0x7f58fc1093f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:21:56.958 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:21:56.957+0000 7f5902c86700 1 --2- 172.21.15.66:0/1989670496 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f58fc0f5410 0x7f58fc1093f0 secure :-1 s=READY pgs=92 cs=0 l=1 rev1=1 rx=0x7f58f0013510 tx=0x7f58f000b040).stop 2023-11-17T05:21:56.958 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:21:56.958+0000 7f5902c86700 1 -- 172.21.15.66:0/1989670496 shutdown_connections 2023-11-17T05:21:56.958 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:21:56.958+0000 7f5902c86700 1 --2- 172.21.15.66:0/1989670496 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f58ec02ff60 0x7f58ec032420 unknown :-1 s=CLOSED pgs=11 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:21:56.959 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:21:56.958+0000 7f5902c86700 1 --2- 172.21.15.66:0/1989670496 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f58fc0f5410 0x7f58fc1093f0 unknown :-1 s=CLOSED pgs=92 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:21:56.959 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:21:56.958+0000 7f5902c86700 1 -- 172.21.15.66:0/1989670496 >> 172.21.15.66:0/1989670496 conn(0x7f58fc0f0b70 msgr2=0x7f58fc0f1660 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:21:56.959 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:21:56.958+0000 7f5902c86700 1 -- 172.21.15.66:0/1989670496 shutdown_connections 2023-11-17T05:21:56.959 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:21:56.958+0000 7f5902c86700 1 -- 172.21.15.66:0/1989670496 wait complete. 2023-11-17T05:21:57.329 INFO:tasks.cephadm:Setting crush tunables to default 2023-11-17T05:21:57.329 DEBUG:teuthology.orchestra.run.smithi040:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.4 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ed6470e2-8508-11ee-8dbe-212e2dc638e7 -- ceph osd crush tunables default 2023-11-17T05:21:58.508 INFO:teuthology.orchestra.run.smithi040.stderr:2023-11-17T05:21:58.506+0000 7faeecf7a700 1 -- 172.21.15.40:0/1678133619 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7faee80f5410 msgr2=0x7faee80f5830 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:21:58.508 INFO:teuthology.orchestra.run.smithi040.stderr:2023-11-17T05:21:58.506+0000 7faeecf7a700 1 --2- 172.21.15.40:0/1678133619 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7faee80f5410 0x7faee80f5830 secure :-1 s=READY pgs=93 cs=0 l=1 rev1=1 rx=0x7faed8009fb0 tx=0x7faed8009b30).stop 2023-11-17T05:21:58.509 INFO:teuthology.orchestra.run.smithi040.stderr:2023-11-17T05:21:58.506+0000 7faeecf7a700 1 -- 172.21.15.40:0/1678133619 shutdown_connections 2023-11-17T05:21:58.509 INFO:teuthology.orchestra.run.smithi040.stderr:2023-11-17T05:21:58.506+0000 7faeecf7a700 1 --2- 172.21.15.40:0/1678133619 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7faee80f5410 0x7faee80f5830 unknown :-1 s=CLOSED pgs=93 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:21:58.509 INFO:teuthology.orchestra.run.smithi040.stderr:2023-11-17T05:21:58.506+0000 7faeecf7a700 1 -- 172.21.15.40:0/1678133619 >> 172.21.15.40:0/1678133619 conn(0x7faee80f0b70 msgr2=0x7faee80f2f90 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:21:58.509 INFO:teuthology.orchestra.run.smithi040.stderr:2023-11-17T05:21:58.506+0000 7faeecf7a700 1 -- 172.21.15.40:0/1678133619 shutdown_connections 2023-11-17T05:21:58.509 INFO:teuthology.orchestra.run.smithi040.stderr:2023-11-17T05:21:58.506+0000 7faeecf7a700 1 -- 172.21.15.40:0/1678133619 wait complete. 2023-11-17T05:21:58.509 INFO:teuthology.orchestra.run.smithi040.stderr:2023-11-17T05:21:58.507+0000 7faeecf7a700 1 Processor -- start 2023-11-17T05:21:58.509 INFO:teuthology.orchestra.run.smithi040.stderr:2023-11-17T05:21:58.507+0000 7faeecf7a700 1 -- start start 2023-11-17T05:21:58.509 INFO:teuthology.orchestra.run.smithi040.stderr:2023-11-17T05:21:58.507+0000 7faeecf7a700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7faee80f5410 0x7faee80fd3a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:21:58.509 INFO:teuthology.orchestra.run.smithi040.stderr:2023-11-17T05:21:58.507+0000 7faeecf7a700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7faee80fb820 con 0x7faee80f5410 2023-11-17T05:21:58.509 INFO:teuthology.orchestra.run.smithi040.stderr:2023-11-17T05:21:58.507+0000 7faee659c700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7faee80f5410 0x7faee80fd3a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:21:58.510 INFO:teuthology.orchestra.run.smithi040.stderr:2023-11-17T05:21:58.507+0000 7faee659c700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7faee80f5410 0x7faee80fd3a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:35020/0 (socket says 172.21.15.40:35020) 2023-11-17T05:21:58.510 INFO:teuthology.orchestra.run.smithi040.stderr:2023-11-17T05:21:58.507+0000 7faee659c700 1 -- 172.21.15.40:0/772238327 learned_addr learned my addr 172.21.15.40:0/772238327 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-11-17T05:21:58.510 INFO:teuthology.orchestra.run.smithi040.stderr:2023-11-17T05:21:58.507+0000 7faee659c700 1 -- 172.21.15.40:0/772238327 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7faed8005040 con 0x7faee80f5410 2023-11-17T05:21:58.510 INFO:teuthology.orchestra.run.smithi040.stderr:2023-11-17T05:21:58.507+0000 7faee659c700 1 --2- 172.21.15.40:0/772238327 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7faee80f5410 0x7faee80fd3a0 secure :-1 s=READY pgs=94 cs=0 l=1 rev1=1 rx=0x7faed8013510 tx=0x7faed800b040).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:21:58.510 INFO:teuthology.orchestra.run.smithi040.stderr:2023-11-17T05:21:58.508+0000 7faed6ffd700 1 -- 172.21.15.40:0/772238327 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7faed8009320 con 0x7faee80f5410 2023-11-17T05:21:58.510 INFO:teuthology.orchestra.run.smithi040.stderr:2023-11-17T05:21:58.508+0000 7faed6ffd700 1 -- 172.21.15.40:0/772238327 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(25 keys) v1 ==== 1074+0+0 (secure 0 0 0) 0x7faed8013f00 con 0x7faee80f5410 2023-11-17T05:21:58.510 INFO:teuthology.orchestra.run.smithi040.stderr:2023-11-17T05:21:58.508+0000 7faeecf7a700 1 -- 172.21.15.40:0/772238327 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7faee80fb9c0 con 0x7faee80f5410 2023-11-17T05:21:58.510 INFO:teuthology.orchestra.run.smithi040.stderr:2023-11-17T05:21:58.508+0000 7faed6ffd700 1 -- 172.21.15.40:0/772238327 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7faed8014d90 con 0x7faee80f5410 2023-11-17T05:21:58.511 INFO:teuthology.orchestra.run.smithi040.stderr:2023-11-17T05:21:58.508+0000 7faeecf7a700 1 -- 172.21.15.40:0/772238327 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7faee80fbe50 con 0x7faee80f5410 2023-11-17T05:21:58.511 INFO:teuthology.orchestra.run.smithi040.stderr:2023-11-17T05:21:58.508+0000 7faed6ffd700 1 -- 172.21.15.40:0/772238327 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 17) v1 ==== 36269+0+0 (secure 0 0 0) 0x7faed80095f0 con 0x7faee80f5410 2023-11-17T05:21:58.511 INFO:teuthology.orchestra.run.smithi040.stderr:2023-11-17T05:21:58.509+0000 7faeecf7a700 1 -- 172.21.15.40:0/772238327 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7faee8059100 con 0x7faee80f5410 2023-11-17T05:21:58.511 INFO:teuthology.orchestra.run.smithi040.stderr:2023-11-17T05:21:58.509+0000 7faed6ffd700 1 --2- 172.21.15.40:0/772238327 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7faed002ffb0 0x7faed0032470 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:21:58.511 INFO:teuthology.orchestra.run.smithi040.stderr:2023-11-17T05:21:58.509+0000 7faed6ffd700 1 -- 172.21.15.40:0/772238327 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(4..4 src has 1..4) v4 ==== 1270+0+0 (secure 0 0 0) 0x7faed8040fe0 con 0x7faee80f5410 2023-11-17T05:21:58.511 INFO:teuthology.orchestra.run.smithi040.stderr:2023-11-17T05:21:58.509+0000 7faee5d9b700 1 --2- 172.21.15.40:0/772238327 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7faed002ffb0 0x7faed0032470 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:21:58.513 INFO:teuthology.orchestra.run.smithi040.stderr:2023-11-17T05:21:58.509+0000 7faee5d9b700 1 --2- 172.21.15.40:0/772238327 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7faed002ffb0 0x7faed0032470 secure :-1 s=READY pgs=12 cs=0 l=1 rev1=1 rx=0x7faedc0096a0 tx=0x7faedc006b40).ready entity=mgr.14164 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:21:58.513 INFO:teuthology.orchestra.run.smithi040.stderr:2023-11-17T05:21:58.513+0000 7faed6ffd700 1 -- 172.21.15.40:0/772238327 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+152212 (secure 0 0 0) 0x7faed806d2b0 con 0x7faee80f5410 2023-11-17T05:21:58.685 INFO:teuthology.orchestra.run.smithi040.stderr:2023-11-17T05:21:58.684+0000 7faeecf7a700 1 -- 172.21.15.40:0/772238327 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "osd crush tunables", "profile": "default"} v 0) v1 -- 0x7faee80fae60 con 0x7faee80f5410 2023-11-17T05:21:58.740 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:21:58 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:21:56.955919+0000 mgr.smithi040.ogcprl (mgr.14164) 21 : audit [DBG] from='client.14184 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-11-17T05:21:58.740 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:21:58 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:21:57.161948+0000 mgr.smithi040.ogcprl (mgr.14164) 22 : cluster [DBG] pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:21:59.287 INFO:teuthology.orchestra.run.smithi040.stderr:2023-11-17T05:21:59.286+0000 7faed6ffd700 1 -- 172.21.15.40:0/772238327 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mon_command_ack([{"prefix": "osd crush tunables", "profile": "default"}]=0 adjusted tunables profile to default v5) v1 ==== 124+0+0 (secure 0 0 0) 0x7faed8014420 con 0x7faee80f5410 2023-11-17T05:21:59.289 INFO:teuthology.orchestra.run.smithi040.stderr:2023-11-17T05:21:59.288+0000 7faeecf7a700 1 -- 172.21.15.40:0/772238327 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7faed002ffb0 msgr2=0x7faed0032470 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:21:59.289 INFO:teuthology.orchestra.run.smithi040.stderr:2023-11-17T05:21:59.288+0000 7faeecf7a700 1 --2- 172.21.15.40:0/772238327 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7faed002ffb0 0x7faed0032470 secure :-1 s=READY pgs=12 cs=0 l=1 rev1=1 rx=0x7faedc0096a0 tx=0x7faedc006b40).stop 2023-11-17T05:21:59.289 INFO:teuthology.orchestra.run.smithi040.stderr:2023-11-17T05:21:59.288+0000 7faeecf7a700 1 -- 172.21.15.40:0/772238327 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7faee80f5410 msgr2=0x7faee80fd3a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:21:59.289 INFO:teuthology.orchestra.run.smithi040.stderr:2023-11-17T05:21:59.288+0000 7faeecf7a700 1 --2- 172.21.15.40:0/772238327 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7faee80f5410 0x7faee80fd3a0 secure :-1 s=READY pgs=94 cs=0 l=1 rev1=1 rx=0x7faed8013510 tx=0x7faed800b040).stop 2023-11-17T05:21:59.289 INFO:teuthology.orchestra.run.smithi040.stderr:2023-11-17T05:21:59.288+0000 7faeecf7a700 1 -- 172.21.15.40:0/772238327 shutdown_connections 2023-11-17T05:21:59.290 INFO:teuthology.orchestra.run.smithi040.stderr:2023-11-17T05:21:59.288+0000 7faeecf7a700 1 --2- 172.21.15.40:0/772238327 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7faed002ffb0 0x7faed0032470 unknown :-1 s=CLOSED pgs=12 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:21:59.290 INFO:teuthology.orchestra.run.smithi040.stderr:2023-11-17T05:21:59.288+0000 7faeecf7a700 1 --2- 172.21.15.40:0/772238327 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7faee80f5410 0x7faee80fd3a0 unknown :-1 s=CLOSED pgs=94 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:21:59.290 INFO:teuthology.orchestra.run.smithi040.stderr:2023-11-17T05:21:59.288+0000 7faeecf7a700 1 -- 172.21.15.40:0/772238327 >> 172.21.15.40:0/772238327 conn(0x7faee80f0b70 msgr2=0x7faee805fe60 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:21:59.290 INFO:teuthology.orchestra.run.smithi040.stderr:2023-11-17T05:21:59.289+0000 7faeecf7a700 1 -- 172.21.15.40:0/772238327 shutdown_connections 2023-11-17T05:21:59.290 INFO:teuthology.orchestra.run.smithi040.stderr:2023-11-17T05:21:59.289+0000 7faeecf7a700 1 -- 172.21.15.40:0/772238327 wait complete. 2023-11-17T05:21:59.290 INFO:teuthology.orchestra.run.smithi040.stderr:adjusted tunables profile to default 2023-11-17T05:21:59.574 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:21:59 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:21:58.685812+0000 mon.smithi040 (mon.0) 118 : audit [INF] from='client.? 172.21.15.40:0/772238327' entity='client.admin' cmd=[{"prefix": "osd crush tunables", "profile": "default"}]: dispatch 2023-11-17T05:21:59.677 INFO:tasks.cephadm:Adding mon.smithi040 on smithi040 2023-11-17T05:21:59.677 INFO:tasks.cephadm:Adding mon.smithi066 on smithi066 2023-11-17T05:21:59.678 DEBUG:teuthology.orchestra.run.smithi066:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.4 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ed6470e2-8508-11ee-8dbe-212e2dc638e7 -- ceph orch apply mon '2;smithi040:172.21.15.40=smithi040;smithi066:172.21.15.66=smithi066' 2023-11-17T05:22:00.739 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:22:00 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:21:59.162337+0000 mgr.smithi040.ogcprl (mgr.14164) 23 : cluster [DBG] pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:22:00.739 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:22:00 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:21:59.286904+0000 mon.smithi040 (mon.0) 119 : audit [INF] from='client.? 172.21.15.40:0/772238327' entity='client.admin' cmd='[{"prefix": "osd crush tunables", "profile": "default"}]': finished 2023-11-17T05:22:00.739 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:22:00 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:21:59.286957+0000 mon.smithi040 (mon.0) 120 : cluster [DBG] osdmap e5: 0 total, 0 up, 0 in 2023-11-17T05:22:01.142 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:01.142+0000 7fafb96cc700 1 -- 172.21.15.66:0/3257993304 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fafb40f5410 msgr2=0x7fafb40f5830 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:22:01.143 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:01.142+0000 7fafb96cc700 1 --2- 172.21.15.66:0/3257993304 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fafb40f5410 0x7fafb40f5830 secure :-1 s=READY pgs=95 cs=0 l=1 rev1=1 rx=0x7faf9c009fb0 tx=0x7faf9c009b30).stop 2023-11-17T05:22:01.143 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:01.142+0000 7fafb96cc700 1 -- 172.21.15.66:0/3257993304 shutdown_connections 2023-11-17T05:22:01.143 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:01.142+0000 7fafb96cc700 1 --2- 172.21.15.66:0/3257993304 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fafb40f5410 0x7fafb40f5830 unknown :-1 s=CLOSED pgs=95 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:22:01.143 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:01.142+0000 7fafb96cc700 1 -- 172.21.15.66:0/3257993304 >> 172.21.15.66:0/3257993304 conn(0x7fafb40f0b70 msgr2=0x7fafb40f2f90 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:22:01.143 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:01.142+0000 7fafb96cc700 1 -- 172.21.15.66:0/3257993304 shutdown_connections 2023-11-17T05:22:01.143 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:01.142+0000 7fafb96cc700 1 -- 172.21.15.66:0/3257993304 wait complete. 2023-11-17T05:22:01.144 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:01.143+0000 7fafb96cc700 1 Processor -- start 2023-11-17T05:22:01.144 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:01.143+0000 7fafb96cc700 1 -- start start 2023-11-17T05:22:01.145 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:01.143+0000 7fafb96cc700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fafb40f5410 0x7fafb4105630 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:22:01.145 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:01.143+0000 7fafb96cc700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fafb4105b20 con 0x7fafb40f5410 2023-11-17T05:22:01.145 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:01.143+0000 7fafb2ffd700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fafb40f5410 0x7fafb4105630 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:22:01.145 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:01.143+0000 7fafb2ffd700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fafb40f5410 0x7fafb4105630 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.66:57992/0 (socket says 172.21.15.66:57992) 2023-11-17T05:22:01.145 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:01.143+0000 7fafb2ffd700 1 -- 172.21.15.66:0/190043030 learned_addr learned my addr 172.21.15.66:0/190043030 (peer_addr_for_me v2:172.21.15.66:0/0) 2023-11-17T05:22:01.145 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:01.143+0000 7fafb2ffd700 1 -- 172.21.15.66:0/190043030 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7faf9c005040 con 0x7fafb40f5410 2023-11-17T05:22:01.145 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:01.143+0000 7fafb2ffd700 1 --2- 172.21.15.66:0/190043030 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fafb40f5410 0x7fafb4105630 secure :-1 s=READY pgs=96 cs=0 l=1 rev1=1 rx=0x7faf9c014000 tx=0x7faf9c00b040).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:22:01.145 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:01.144+0000 7fafab7fe700 1 -- 172.21.15.66:0/190043030 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7faf9c0095f0 con 0x7fafb40f5410 2023-11-17T05:22:01.146 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:01.144+0000 7fafb96cc700 1 -- 172.21.15.66:0/190043030 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fafb4105d20 con 0x7fafb40f5410 2023-11-17T05:22:01.146 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:01.144+0000 7fafab7fe700 1 -- 172.21.15.66:0/190043030 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(25 keys) v1 ==== 1074+0+0 (secure 0 0 0) 0x7faf9c007e70 con 0x7fafb40f5410 2023-11-17T05:22:01.146 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:01.144+0000 7fafab7fe700 1 -- 172.21.15.66:0/190043030 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7faf9c014ea0 con 0x7fafb40f5410 2023-11-17T05:22:01.146 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:01.144+0000 7fafb96cc700 1 -- 172.21.15.66:0/190043030 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fafb41061b0 con 0x7fafb40f5410 2023-11-17T05:22:01.147 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:01.145+0000 7fafab7fe700 1 -- 172.21.15.66:0/190043030 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 17) v1 ==== 36269+0+0 (secure 0 0 0) 0x7faf9c025070 con 0x7fafb40f5410 2023-11-17T05:22:01.147 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:01.145+0000 7fafb96cc700 1 -- 172.21.15.66:0/190043030 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fafb4045ad0 con 0x7fafb40f5410 2023-11-17T05:22:01.147 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:01.145+0000 7fafab7fe700 1 --2- 172.21.15.66:0/190043030 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fafa002ff60 0x7fafa0032420 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:22:01.147 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:01.145+0000 7fafab7fe700 1 -- 172.21.15.66:0/190043030 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(5..5 src has 1..5) v4 ==== 1270+0+0 (secure 0 0 0) 0x7faf9c040940 con 0x7fafb40f5410 2023-11-17T05:22:01.147 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:01.145+0000 7fafb27fc700 1 --2- 172.21.15.66:0/190043030 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fafa002ff60 0x7fafa0032420 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:22:01.147 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:01.145+0000 7fafb27fc700 1 --2- 172.21.15.66:0/190043030 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fafa002ff60 0x7fafa0032420 secure :-1 s=READY pgs=13 cs=0 l=1 rev1=1 rx=0x7fafa40096a0 tx=0x7fafa4006b40).ready entity=mgr.14164 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:22:01.150 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:01.150+0000 7fafab7fe700 1 -- 172.21.15.66:0/190043030 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+152212 (secure 0 0 0) 0x7faf9c0079c0 con 0x7fafb40f5410 2023-11-17T05:22:01.341 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:01.339+0000 7fafb96cc700 1 -- 172.21.15.66:0/190043030 --> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] -- mgr_command(tid 0: {"prefix": "orch apply", "service_type": "mon", "placement": "2;smithi040:172.21.15.40=smithi040;smithi066:172.21.15.66=smithi066", "target": ["mon-mgr", ""]}) v1 -- 0x7fafb40f1e20 con 0x7fafa002ff60 2023-11-17T05:22:01.346 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:01.345+0000 7fafab7fe700 1 -- 172.21.15.66:0/190043030 <== mgr.14164 v2:172.21.15.40:6800/1119389001 1 ==== mgr_command_reply(tid 0: 0 ) v1 ==== 8+0+24 (secure 0 0 0) 0x7fafb40f1e20 con 0x7fafa002ff60 2023-11-17T05:22:01.347 INFO:teuthology.orchestra.run.smithi066.stdout:Scheduled mon update... 2023-11-17T05:22:01.348 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:01.346+0000 7fafb96cc700 1 -- 172.21.15.66:0/190043030 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fafa002ff60 msgr2=0x7fafa0032420 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:22:01.348 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:01.346+0000 7fafb96cc700 1 --2- 172.21.15.66:0/190043030 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fafa002ff60 0x7fafa0032420 secure :-1 s=READY pgs=13 cs=0 l=1 rev1=1 rx=0x7fafa40096a0 tx=0x7fafa4006b40).stop 2023-11-17T05:22:01.349 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:01.346+0000 7fafb96cc700 1 -- 172.21.15.66:0/190043030 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fafb40f5410 msgr2=0x7fafb4105630 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:22:01.349 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:01.346+0000 7fafb96cc700 1 --2- 172.21.15.66:0/190043030 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fafb40f5410 0x7fafb4105630 secure :-1 s=READY pgs=96 cs=0 l=1 rev1=1 rx=0x7faf9c014000 tx=0x7faf9c00b040).stop 2023-11-17T05:22:01.349 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:01.346+0000 7fafb96cc700 1 -- 172.21.15.66:0/190043030 shutdown_connections 2023-11-17T05:22:01.349 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:01.346+0000 7fafb96cc700 1 --2- 172.21.15.66:0/190043030 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fafa002ff60 0x7fafa0032420 unknown :-1 s=CLOSED pgs=13 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:22:01.349 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:01.346+0000 7fafb96cc700 1 --2- 172.21.15.66:0/190043030 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fafb40f5410 0x7fafb4105630 unknown :-1 s=CLOSED pgs=96 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:22:01.349 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:01.346+0000 7fafb96cc700 1 -- 172.21.15.66:0/190043030 >> 172.21.15.66:0/190043030 conn(0x7fafb40f0b70 msgr2=0x7fafb40f1660 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:22:01.350 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:01.346+0000 7fafb96cc700 1 -- 172.21.15.66:0/190043030 shutdown_connections 2023-11-17T05:22:01.350 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:01.346+0000 7fafb96cc700 1 -- 172.21.15.66:0/190043030 wait complete. 2023-11-17T05:22:01.727 DEBUG:teuthology.orchestra.run.smithi066:mon.smithi066> sudo journalctl -f -n 0 -u ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7@mon.smithi066.service 2023-11-17T05:22:01.732 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2023-11-17T05:22:01.732 DEBUG:teuthology.orchestra.run.smithi066:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.4 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ed6470e2-8508-11ee-8dbe-212e2dc638e7 -- ceph mon dump -f json 2023-11-17T05:22:01.761 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:-- Logs begin at Fri 2023-11-17 05:03:13 UTC. -- 2023-11-17T05:22:02.739 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:22:02 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:22:01.162714+0000 mgr.smithi040.ogcprl (mgr.14164) 24 : cluster [DBG] pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:22:02.739 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:22:02 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:22:01.340722+0000 mgr.smithi040.ogcprl (mgr.14164) 25 : audit [DBG] from='client.14188 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "placement": "2;smithi040:172.21.15.40=smithi040;smithi066:172.21.15.66=smithi066", "target": ["mon-mgr", ""]}]: dispatch 2023-11-17T05:22:02.739 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:22:02 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cephadm 2023-11-17T05:22:01.342162+0000 mgr.smithi040.ogcprl (mgr.14164) 26 : cephadm [INF] Saving service mon spec with placement smithi040:172.21.15.40=smithi040;smithi066:172.21.15.66=smithi066;count:2 2023-11-17T05:22:02.740 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:22:02 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:22:01.344688+0000 mon.smithi040 (mon.0) 121 : audit [INF] from='mgr.14164 172.21.15.40:0/2203032451' entity='mgr.smithi040.ogcprl' 2023-11-17T05:22:02.851 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:02.850+0000 7fef55cb7700 1 -- 172.21.15.66:0/3280582333 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fef500f6510 msgr2=0x7fef500f6930 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:22:02.851 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:02.850+0000 7fef55cb7700 1 --2- 172.21.15.66:0/3280582333 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fef500f6510 0x7fef500f6930 secure :-1 s=READY pgs=97 cs=0 l=1 rev1=1 rx=0x7fef40009fb0 tx=0x7fef40009b30).stop 2023-11-17T05:22:02.851 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:02.850+0000 7fef55cb7700 1 -- 172.21.15.66:0/3280582333 shutdown_connections 2023-11-17T05:22:02.851 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:02.850+0000 7fef55cb7700 1 --2- 172.21.15.66:0/3280582333 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fef500f6510 0x7fef500f6930 unknown :-1 s=CLOSED pgs=97 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:22:02.852 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:02.850+0000 7fef55cb7700 1 -- 172.21.15.66:0/3280582333 >> 172.21.15.66:0/3280582333 conn(0x7fef500f1c70 msgr2=0x7fef500f4090 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:22:02.852 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:02.850+0000 7fef55cb7700 1 -- 172.21.15.66:0/3280582333 shutdown_connections 2023-11-17T05:22:02.852 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:02.850+0000 7fef55cb7700 1 -- 172.21.15.66:0/3280582333 wait complete. 2023-11-17T05:22:02.852 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:02.851+0000 7fef55cb7700 1 Processor -- start 2023-11-17T05:22:02.852 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:02.851+0000 7fef55cb7700 1 -- start start 2023-11-17T05:22:02.852 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:02.851+0000 7fef55cb7700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fef500f6510 0x7fef5006b9a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:22:02.852 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:02.851+0000 7fef55cb7700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fef50069e20 con 0x7fef500f6510 2023-11-17T05:22:02.853 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:02.851+0000 7fef4f7fe700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fef500f6510 0x7fef5006b9a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:22:02.854 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:02.851+0000 7fef4f7fe700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fef500f6510 0x7fef5006b9a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.66:36590/0 (socket says 172.21.15.66:36590) 2023-11-17T05:22:02.854 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:02.851+0000 7fef4f7fe700 1 -- 172.21.15.66:0/2438090324 learned_addr learned my addr 172.21.15.66:0/2438090324 (peer_addr_for_me v2:172.21.15.66:0/0) 2023-11-17T05:22:02.854 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:02.852+0000 7fef4f7fe700 1 -- 172.21.15.66:0/2438090324 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fef40005040 con 0x7fef500f6510 2023-11-17T05:22:02.854 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:02.852+0000 7fef4f7fe700 1 --2- 172.21.15.66:0/2438090324 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fef500f6510 0x7fef5006b9a0 secure :-1 s=READY pgs=98 cs=0 l=1 rev1=1 rx=0x7fef40013510 tx=0x7fef4000b040).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:22:02.854 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:02.852+0000 7fef47fff700 1 -- 172.21.15.66:0/2438090324 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7fef40009320 con 0x7fef500f6510 2023-11-17T05:22:02.854 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:02.852+0000 7fef55cb7700 1 -- 172.21.15.66:0/2438090324 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fef50069fc0 con 0x7fef500f6510 2023-11-17T05:22:02.854 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:02.852+0000 7fef47fff700 1 -- 172.21.15.66:0/2438090324 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(25 keys) v1 ==== 1074+0+0 (secure 0 0 0) 0x7fef40013f00 con 0x7fef500f6510 2023-11-17T05:22:02.854 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:02.852+0000 7fef47fff700 1 -- 172.21.15.66:0/2438090324 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7fef40014d90 con 0x7fef500f6510 2023-11-17T05:22:02.854 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:02.852+0000 7fef55cb7700 1 -- 172.21.15.66:0/2438090324 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fef5006a450 con 0x7fef500f6510 2023-11-17T05:22:02.854 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:02.853+0000 7fef47fff700 1 -- 172.21.15.66:0/2438090324 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 17) v1 ==== 36269+0+0 (secure 0 0 0) 0x7fef400095f0 con 0x7fef500f6510 2023-11-17T05:22:02.854 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:02.853+0000 7fef55cb7700 1 -- 172.21.15.66:0/2438090324 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fef50059100 con 0x7fef500f6510 2023-11-17T05:22:02.854 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:02.853+0000 7fef47fff700 1 --2- 172.21.15.66:0/2438090324 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fef3c02ff60 0x7fef3c032420 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:22:02.855 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:02.853+0000 7fef47fff700 1 -- 172.21.15.66:0/2438090324 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(5..5 src has 1..5) v4 ==== 1270+0+0 (secure 0 0 0) 0x7fef400410c0 con 0x7fef500f6510 2023-11-17T05:22:02.855 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:02.853+0000 7fef4effd700 1 --2- 172.21.15.66:0/2438090324 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fef3c02ff60 0x7fef3c032420 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:22:02.855 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:02.854+0000 7fef4effd700 1 --2- 172.21.15.66:0/2438090324 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fef3c02ff60 0x7fef3c032420 secure :-1 s=READY pgs=14 cs=0 l=1 rev1=1 rx=0x7fef380096a0 tx=0x7fef38006b40).ready entity=mgr.14164 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:22:02.862 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:02.858+0000 7fef47fff700 1 -- 172.21.15.66:0/2438090324 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+152212 (secure 0 0 0) 0x7fef40014420 con 0x7fef500f6510 2023-11-17T05:22:03.075 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:03.075+0000 7fef55cb7700 1 -- 172.21.15.66:0/2438090324 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "mon dump", "format": "json"} v 0) v1 -- 0x7fef500fbf60 con 0x7fef500f6510 2023-11-17T05:22:03.075 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:03.075+0000 7fef47fff700 1 -- 172.21.15.66:0/2438090324 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mon_command_ack([{"prefix": "mon dump", "format": "json"}]=0 dumped monmap epoch 1 v1) v1 ==== 95+0+690 (secure 0 0 0) 0x7fef40010230 con 0x7fef500f6510 2023-11-17T05:22:03.076 INFO:teuthology.orchestra.run.smithi066.stdout: 2023-11-17T05:22:03.076 INFO:teuthology.orchestra.run.smithi066.stdout:{"epoch":1,"fsid":"ed6470e2-8508-11ee-8dbe-212e2dc638e7","modified":"2023-11-17T05:20:13.934038Z","created":"2023-11-17T05:20:13.934038Z","min_mon_release":16,"min_mon_release_name":"pacific","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging"],"optional":[]},"mons":[{"rank":0,"name":"smithi040","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:3300","nonce":0},{"type":"v1","addr":"172.21.15.40:6789","nonce":0}]},"addr":"172.21.15.40:6789/0","public_addr":"172.21.15.40:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2023-11-17T05:22:03.078 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:03.077+0000 7fef55cb7700 1 -- 172.21.15.66:0/2438090324 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fef3c02ff60 msgr2=0x7fef3c032420 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:22:03.078 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:03.077+0000 7fef55cb7700 1 --2- 172.21.15.66:0/2438090324 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fef3c02ff60 0x7fef3c032420 secure :-1 s=READY pgs=14 cs=0 l=1 rev1=1 rx=0x7fef380096a0 tx=0x7fef38006b40).stop 2023-11-17T05:22:03.078 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:03.077+0000 7fef55cb7700 1 -- 172.21.15.66:0/2438090324 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fef500f6510 msgr2=0x7fef5006b9a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:22:03.078 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:03.077+0000 7fef55cb7700 1 --2- 172.21.15.66:0/2438090324 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fef500f6510 0x7fef5006b9a0 secure :-1 s=READY pgs=98 cs=0 l=1 rev1=1 rx=0x7fef40013510 tx=0x7fef4000b040).stop 2023-11-17T05:22:03.078 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:03.077+0000 7fef55cb7700 1 -- 172.21.15.66:0/2438090324 shutdown_connections 2023-11-17T05:22:03.078 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:03.077+0000 7fef55cb7700 1 --2- 172.21.15.66:0/2438090324 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fef3c02ff60 0x7fef3c032420 unknown :-1 s=CLOSED pgs=14 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:22:03.078 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:03.077+0000 7fef55cb7700 1 --2- 172.21.15.66:0/2438090324 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fef500f6510 0x7fef5006b9a0 unknown :-1 s=CLOSED pgs=98 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:22:03.078 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:03.077+0000 7fef55cb7700 1 -- 172.21.15.66:0/2438090324 >> 172.21.15.66:0/2438090324 conn(0x7fef500f1c70 msgr2=0x7fef500f2760 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:22:03.078 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:03.077+0000 7fef55cb7700 1 -- 172.21.15.66:0/2438090324 shutdown_connections 2023-11-17T05:22:03.078 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:03.077+0000 7fef55cb7700 1 -- 172.21.15.66:0/2438090324 wait complete. 2023-11-17T05:22:03.079 INFO:teuthology.orchestra.run.smithi066.stderr:dumped monmap epoch 1 2023-11-17T05:22:03.739 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:22:03 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:22:03.076040+0000 mon.smithi040 (mon.0) 122 : audit [DBG] from='client.? 172.21.15.66:0/2438090324' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-11-17T05:22:04.448 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2023-11-17T05:22:04.448 DEBUG:teuthology.orchestra.run.smithi066:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.4 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ed6470e2-8508-11ee-8dbe-212e2dc638e7 -- ceph mon dump -f json 2023-11-17T05:22:04.739 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:22:04 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:22:03.163067+0000 mgr.smithi040.ogcprl (mgr.14164) 27 : cluster [DBG] pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:22:05.589 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:05.589+0000 7fc47451c700 1 -- 172.21.15.66:0/2865981188 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fc46c0f5410 msgr2=0x7fc46c0f5830 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:22:05.589 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:05.589+0000 7fc47451c700 1 --2- 172.21.15.66:0/2865981188 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fc46c0f5410 0x7fc46c0f5830 secure :-1 s=READY pgs=99 cs=0 l=1 rev1=1 rx=0x7fc468009fb0 tx=0x7fc468009b30).stop 2023-11-17T05:22:05.589 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:05.589+0000 7fc47451c700 1 -- 172.21.15.66:0/2865981188 shutdown_connections 2023-11-17T05:22:05.589 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:05.589+0000 7fc47451c700 1 --2- 172.21.15.66:0/2865981188 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fc46c0f5410 0x7fc46c0f5830 unknown :-1 s=CLOSED pgs=99 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:22:05.589 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:05.589+0000 7fc47451c700 1 -- 172.21.15.66:0/2865981188 >> 172.21.15.66:0/2865981188 conn(0x7fc46c0f0b70 msgr2=0x7fc46c0f2f90 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:22:05.590 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:05.589+0000 7fc47451c700 1 -- 172.21.15.66:0/2865981188 shutdown_connections 2023-11-17T05:22:05.590 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:05.589+0000 7fc47451c700 1 -- 172.21.15.66:0/2865981188 wait complete. 2023-11-17T05:22:05.591 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:05.589+0000 7fc47451c700 1 Processor -- start 2023-11-17T05:22:05.591 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:05.589+0000 7fc47451c700 1 -- start start 2023-11-17T05:22:05.591 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:05.589+0000 7fc47451c700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fc46c0f5410 0x7fc46c109450 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:22:05.591 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:05.589+0000 7fc47451c700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fc46c060220 con 0x7fc46c0f5410 2023-11-17T05:22:05.591 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:05.590+0000 7fc4722b8700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fc46c0f5410 0x7fc46c109450 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:22:05.591 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:05.590+0000 7fc4722b8700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fc46c0f5410 0x7fc46c109450 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.66:36612/0 (socket says 172.21.15.66:36612) 2023-11-17T05:22:05.591 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:05.590+0000 7fc4722b8700 1 -- 172.21.15.66:0/1400128313 learned_addr learned my addr 172.21.15.66:0/1400128313 (peer_addr_for_me v2:172.21.15.66:0/0) 2023-11-17T05:22:05.591 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:05.590+0000 7fc4722b8700 1 -- 172.21.15.66:0/1400128313 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fc468005040 con 0x7fc46c0f5410 2023-11-17T05:22:05.591 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:05.590+0000 7fc4722b8700 1 --2- 172.21.15.66:0/1400128313 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fc46c0f5410 0x7fc46c109450 secure :-1 s=READY pgs=100 cs=0 l=1 rev1=1 rx=0x7fc468013510 tx=0x7fc46800b040).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:22:05.592 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:05.590+0000 7fc45effd700 1 -- 172.21.15.66:0/1400128313 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7fc468009320 con 0x7fc46c0f5410 2023-11-17T05:22:05.592 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:05.590+0000 7fc47451c700 1 -- 172.21.15.66:0/1400128313 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fc46c060420 con 0x7fc46c0f5410 2023-11-17T05:22:05.592 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:05.590+0000 7fc45effd700 1 -- 172.21.15.66:0/1400128313 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(25 keys) v1 ==== 1074+0+0 (secure 0 0 0) 0x7fc468013f00 con 0x7fc46c0f5410 2023-11-17T05:22:05.592 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:05.590+0000 7fc45effd700 1 -- 172.21.15.66:0/1400128313 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7fc468014d90 con 0x7fc46c0f5410 2023-11-17T05:22:05.592 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:05.590+0000 7fc47451c700 1 -- 172.21.15.66:0/1400128313 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fc46c109b00 con 0x7fc46c0f5410 2023-11-17T05:22:05.593 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:05.591+0000 7fc45effd700 1 -- 172.21.15.66:0/1400128313 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 17) v1 ==== 36269+0+0 (secure 0 0 0) 0x7fc4680095f0 con 0x7fc46c0f5410 2023-11-17T05:22:05.593 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:05.591+0000 7fc47451c700 1 -- 172.21.15.66:0/1400128313 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fc46c059100 con 0x7fc46c0f5410 2023-11-17T05:22:05.593 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:05.591+0000 7fc45effd700 1 --2- 172.21.15.66:0/1400128313 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fc45802ff60 0x7fc458032420 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:22:05.593 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:05.591+0000 7fc45effd700 1 -- 172.21.15.66:0/1400128313 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(5..5 src has 1..5) v4 ==== 1270+0+0 (secure 0 0 0) 0x7fc468041000 con 0x7fc46c0f5410 2023-11-17T05:22:05.593 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:05.592+0000 7fc471ab7700 1 --2- 172.21.15.66:0/1400128313 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fc45802ff60 0x7fc458032420 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:22:05.593 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:05.592+0000 7fc471ab7700 1 --2- 172.21.15.66:0/1400128313 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fc45802ff60 0x7fc458032420 secure :-1 s=READY pgs=15 cs=0 l=1 rev1=1 rx=0x7fc4600096a0 tx=0x7fc460006b40).ready entity=mgr.14164 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:22:05.596 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:05.596+0000 7fc45effd700 1 -- 172.21.15.66:0/1400128313 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+152212 (secure 0 0 0) 0x7fc468014420 con 0x7fc46c0f5410 2023-11-17T05:22:05.816 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:05.815+0000 7fc47451c700 1 -- 172.21.15.66:0/1400128313 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "mon dump", "format": "json"} v 0) v1 -- 0x7fc46c045ad0 con 0x7fc46c0f5410 2023-11-17T05:22:05.816 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:05.816+0000 7fc45effd700 1 -- 172.21.15.66:0/1400128313 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mon_command_ack([{"prefix": "mon dump", "format": "json"}]=0 dumped monmap epoch 1 v1) v1 ==== 95+0+690 (secure 0 0 0) 0x7fc468010230 con 0x7fc46c0f5410 2023-11-17T05:22:05.817 INFO:teuthology.orchestra.run.smithi066.stdout: 2023-11-17T05:22:05.817 INFO:teuthology.orchestra.run.smithi066.stdout:{"epoch":1,"fsid":"ed6470e2-8508-11ee-8dbe-212e2dc638e7","modified":"2023-11-17T05:20:13.934038Z","created":"2023-11-17T05:20:13.934038Z","min_mon_release":16,"min_mon_release_name":"pacific","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging"],"optional":[]},"mons":[{"rank":0,"name":"smithi040","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:3300","nonce":0},{"type":"v1","addr":"172.21.15.40:6789","nonce":0}]},"addr":"172.21.15.40:6789/0","public_addr":"172.21.15.40:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2023-11-17T05:22:05.818 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:05.818+0000 7fc47451c700 1 -- 172.21.15.66:0/1400128313 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fc45802ff60 msgr2=0x7fc458032420 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:22:05.818 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:05.818+0000 7fc47451c700 1 --2- 172.21.15.66:0/1400128313 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fc45802ff60 0x7fc458032420 secure :-1 s=READY pgs=15 cs=0 l=1 rev1=1 rx=0x7fc4600096a0 tx=0x7fc460006b40).stop 2023-11-17T05:22:05.819 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:05.818+0000 7fc47451c700 1 -- 172.21.15.66:0/1400128313 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fc46c0f5410 msgr2=0x7fc46c109450 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:22:05.819 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:05.818+0000 7fc47451c700 1 --2- 172.21.15.66:0/1400128313 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fc46c0f5410 0x7fc46c109450 secure :-1 s=READY pgs=100 cs=0 l=1 rev1=1 rx=0x7fc468013510 tx=0x7fc46800b040).stop 2023-11-17T05:22:05.819 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:05.818+0000 7fc47451c700 1 -- 172.21.15.66:0/1400128313 shutdown_connections 2023-11-17T05:22:05.819 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:05.818+0000 7fc47451c700 1 --2- 172.21.15.66:0/1400128313 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fc45802ff60 0x7fc458032420 unknown :-1 s=CLOSED pgs=15 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:22:05.819 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:05.818+0000 7fc47451c700 1 --2- 172.21.15.66:0/1400128313 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fc46c0f5410 0x7fc46c109450 unknown :-1 s=CLOSED pgs=100 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:22:05.819 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:05.818+0000 7fc47451c700 1 -- 172.21.15.66:0/1400128313 >> 172.21.15.66:0/1400128313 conn(0x7fc46c0f0b70 msgr2=0x7fc46c0f1660 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:22:05.819 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:05.818+0000 7fc47451c700 1 -- 172.21.15.66:0/1400128313 shutdown_connections 2023-11-17T05:22:05.819 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:05.818+0000 7fc47451c700 1 -- 172.21.15.66:0/1400128313 wait complete. 2023-11-17T05:22:05.819 INFO:teuthology.orchestra.run.smithi066.stderr:dumped monmap epoch 1 2023-11-17T05:22:06.739 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:22:06 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:22:05.163452+0000 mgr.smithi040.ogcprl (mgr.14164) 28 : cluster [DBG] pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:22:06.739 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:22:06 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:22:05.816820+0000 mon.smithi040 (mon.0) 123 : audit [DBG] from='client.? 172.21.15.66:0/1400128313' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-11-17T05:22:07.190 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2023-11-17T05:22:31.323 DEBUG:teuthology.orchestra.run.smithi066:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.4 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ed6470e2-8508-11ee-8dbe-212e2dc638e7 -- ceph mon dump -f json 2023-11-17T05:22:31.325 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:22:08 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:22:07.163802+0000 mgr.smithi040.ogcprl (mgr.14164) 29 : cluster [DBG] pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:22:31.325 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:22:10 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:22:09.164086+0000 mgr.smithi040.ogcprl (mgr.14164) 30 : cluster [DBG] pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:22:31.325 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:22:11 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:22:11.307432+0000 mon.smithi040 (mon.0) 124 : audit [INF] from='mgr.14164 172.21.15.40:0/2203032451' entity='mgr.smithi040.ogcprl' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi040.ogcprl/mirror_snapshot_schedule"}]: dispatch 2023-11-17T05:22:31.325 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:22:11 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:22:11.310162+0000 mon.smithi040 (mon.0) 125 : audit [INF] from='mgr.14164 172.21.15.40:0/2203032451' entity='mgr.smithi040.ogcprl' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi040.ogcprl/trash_purge_schedule"}]: dispatch 2023-11-17T05:22:31.326 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:22:12 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:22:11.164321+0000 mgr.smithi040.ogcprl (mgr.14164) 31 : cluster [DBG] pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:22:31.326 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:22:14 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:22:13.164731+0000 mgr.smithi040.ogcprl (mgr.14164) 32 : cluster [DBG] pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:22:31.326 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:22:16 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:22:15.165005+0000 mgr.smithi040.ogcprl (mgr.14164) 33 : cluster [DBG] pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:22:31.326 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:22:18 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:22:17.165329+0000 mgr.smithi040.ogcprl (mgr.14164) 34 : cluster [DBG] pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:22:31.326 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:22:20 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:22:19.165720+0000 mgr.smithi040.ogcprl (mgr.14164) 35 : cluster [DBG] pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:22:31.326 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:22:22 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:22:21.165940+0000 mgr.smithi040.ogcprl (mgr.14164) 36 : cluster [DBG] pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:22:31.326 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:22:24 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:22:23.166188+0000 mgr.smithi040.ogcprl (mgr.14164) 37 : cluster [DBG] pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:22:31.326 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:22:26 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:22:25.166452+0000 mgr.smithi040.ogcprl (mgr.14164) 38 : cluster [DBG] pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:22:31.326 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:22:28 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:22:27.166761+0000 mgr.smithi040.ogcprl (mgr.14164) 39 : cluster [DBG] pgmap v32: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:22:31.326 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:22:30 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:22:29.167068+0000 mgr.smithi040.ogcprl (mgr.14164) 40 : cluster [DBG] pgmap v33: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:22:32.557 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:32.554+0000 7fc80d35d700 1 -- 172.21.15.66:0/2583540186 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fc8080f51a0 msgr2=0x7fc8080f55c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:22:32.733 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:32.554+0000 7fc80d35d700 1 --2- 172.21.15.66:0/2583540186 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fc8080f51a0 0x7fc8080f55c0 secure :-1 s=READY pgs=101 cs=0 l=1 rev1=1 rx=0x7fc7f8009fb0 tx=0x7fc7f8009b30).stop 2023-11-17T05:22:32.733 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:32.554+0000 7fc80d35d700 1 -- 172.21.15.66:0/2583540186 shutdown_connections 2023-11-17T05:22:32.734 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:32.554+0000 7fc80d35d700 1 --2- 172.21.15.66:0/2583540186 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fc8080f51a0 0x7fc8080f55c0 unknown :-1 s=CLOSED pgs=101 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:22:32.734 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:32.554+0000 7fc80d35d700 1 -- 172.21.15.66:0/2583540186 >> 172.21.15.66:0/2583540186 conn(0x7fc8080f09a0 msgr2=0x7fc8080f2dc0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:22:32.734 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:32.554+0000 7fc80d35d700 1 -- 172.21.15.66:0/2583540186 shutdown_connections 2023-11-17T05:22:32.734 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:32.554+0000 7fc80d35d700 1 -- 172.21.15.66:0/2583540186 wait complete. 2023-11-17T05:22:32.734 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:32.554+0000 7fc80d35d700 1 Processor -- start 2023-11-17T05:22:32.734 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:32.555+0000 7fc80d35d700 1 -- start start 2023-11-17T05:22:32.734 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:32.555+0000 7fc80d35d700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fc8080f51a0 0x7fc808106f60 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:22:32.734 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:32.555+0000 7fc80d35d700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fc808060280 con 0x7fc8080f51a0 2023-11-17T05:22:32.734 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:32.555+0000 7fc807fff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fc8080f51a0 0x7fc808106f60 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:22:32.734 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:32.555+0000 7fc807fff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fc8080f51a0 0x7fc808106f60 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.66:36322/0 (socket says 172.21.15.66:36322) 2023-11-17T05:22:32.734 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:32.555+0000 7fc807fff700 1 -- 172.21.15.66:0/2341386068 learned_addr learned my addr 172.21.15.66:0/2341386068 (peer_addr_for_me v2:172.21.15.66:0/0) 2023-11-17T05:22:32.734 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:32.555+0000 7fc807fff700 1 -- 172.21.15.66:0/2341386068 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fc7f8005040 con 0x7fc8080f51a0 2023-11-17T05:22:32.735 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:32.555+0000 7fc807fff700 1 --2- 172.21.15.66:0/2341386068 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fc8080f51a0 0x7fc808106f60 secure :-1 s=READY pgs=102 cs=0 l=1 rev1=1 rx=0x7fc7f8013510 tx=0x7fc7f800b040).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:22:32.735 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:32.556+0000 7fc804ff9700 1 -- 172.21.15.66:0/2341386068 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7fc7f8009320 con 0x7fc8080f51a0 2023-11-17T05:22:32.735 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:32.556+0000 7fc80d35d700 1 -- 172.21.15.66:0/2341386068 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fc808060480 con 0x7fc8080f51a0 2023-11-17T05:22:32.735 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:32.556+0000 7fc804ff9700 1 -- 172.21.15.66:0/2341386068 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(25 keys) v1 ==== 1074+0+0 (secure 0 0 0) 0x7fc7f8013f00 con 0x7fc8080f51a0 2023-11-17T05:22:32.735 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:32.556+0000 7fc804ff9700 1 -- 172.21.15.66:0/2341386068 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7fc7f8014dc0 con 0x7fc8080f51a0 2023-11-17T05:22:32.735 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:32.556+0000 7fc80d35d700 1 -- 172.21.15.66:0/2341386068 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fc808107690 con 0x7fc8080f51a0 2023-11-17T05:22:32.736 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:32.556+0000 7fc804ff9700 1 -- 172.21.15.66:0/2341386068 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 17) v1 ==== 36269+0+0 (secure 0 0 0) 0x7fc7f80095f0 con 0x7fc8080f51a0 2023-11-17T05:22:32.736 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:32.556+0000 7fc80d35d700 1 -- 172.21.15.66:0/2341386068 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fc808045ad0 con 0x7fc8080f51a0 2023-11-17T05:22:32.736 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:32.557+0000 7fc804ff9700 1 --2- 172.21.15.66:0/2341386068 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fc7f002ff60 0x7fc7f0032420 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:22:32.736 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:32.557+0000 7fc804ff9700 1 -- 172.21.15.66:0/2341386068 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(5..5 src has 1..5) v4 ==== 1270+0+0 (secure 0 0 0) 0x7fc7f8041000 con 0x7fc8080f51a0 2023-11-17T05:22:32.736 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:32.557+0000 7fc8077fe700 1 --2- 172.21.15.66:0/2341386068 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fc7f002ff60 0x7fc7f0032420 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:22:32.736 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:32.557+0000 7fc8077fe700 1 --2- 172.21.15.66:0/2341386068 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fc7f002ff60 0x7fc7f0032420 secure :-1 s=READY pgs=16 cs=0 l=1 rev1=1 rx=0x7fc7fc0096a0 tx=0x7fc7fc006b40).ready entity=mgr.14164 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:22:32.737 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:32.562+0000 7fc804ff9700 1 -- 172.21.15.66:0/2341386068 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+152212 (secure 0 0 0) 0x7fc7f8014420 con 0x7fc8080f51a0 2023-11-17T05:22:33.004 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:22:32 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:22:31.167427+0000 mgr.smithi040.ogcprl (mgr.14164) 41 : cluster [DBG] pgmap v34: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:22:33.004 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:32.779+0000 7fc80d35d700 1 -- 172.21.15.66:0/2341386068 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "mon dump", "format": "json"} v 0) v1 -- 0x7fc8080fabe0 con 0x7fc8080f51a0 2023-11-17T05:22:33.004 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:32.780+0000 7fc804ff9700 1 -- 172.21.15.66:0/2341386068 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mon_command_ack([{"prefix": "mon dump", "format": "json"}]=0 dumped monmap epoch 1 v1) v1 ==== 95+0+690 (secure 0 0 0) 0x7fc7f8010230 con 0x7fc8080f51a0 2023-11-17T05:22:33.005 INFO:teuthology.orchestra.run.smithi066.stdout: 2023-11-17T05:22:33.005 INFO:teuthology.orchestra.run.smithi066.stdout:{"epoch":1,"fsid":"ed6470e2-8508-11ee-8dbe-212e2dc638e7","modified":"2023-11-17T05:20:13.934038Z","created":"2023-11-17T05:20:13.934038Z","min_mon_release":16,"min_mon_release_name":"pacific","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging"],"optional":[]},"mons":[{"rank":0,"name":"smithi040","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:3300","nonce":0},{"type":"v1","addr":"172.21.15.40:6789","nonce":0}]},"addr":"172.21.15.40:6789/0","public_addr":"172.21.15.40:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2023-11-17T05:22:33.005 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:32.782+0000 7fc80d35d700 1 -- 172.21.15.66:0/2341386068 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fc7f002ff60 msgr2=0x7fc7f0032420 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:22:33.005 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:32.782+0000 7fc80d35d700 1 --2- 172.21.15.66:0/2341386068 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fc7f002ff60 0x7fc7f0032420 secure :-1 s=READY pgs=16 cs=0 l=1 rev1=1 rx=0x7fc7fc0096a0 tx=0x7fc7fc006b40).stop 2023-11-17T05:22:33.005 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:32.782+0000 7fc80d35d700 1 -- 172.21.15.66:0/2341386068 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fc8080f51a0 msgr2=0x7fc808106f60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:22:33.006 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:32.782+0000 7fc80d35d700 1 --2- 172.21.15.66:0/2341386068 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fc8080f51a0 0x7fc808106f60 secure :-1 s=READY pgs=102 cs=0 l=1 rev1=1 rx=0x7fc7f8013510 tx=0x7fc7f800b040).stop 2023-11-17T05:22:33.006 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:32.782+0000 7fc80d35d700 1 -- 172.21.15.66:0/2341386068 shutdown_connections 2023-11-17T05:22:33.006 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:32.782+0000 7fc80d35d700 1 --2- 172.21.15.66:0/2341386068 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fc7f002ff60 0x7fc7f0032420 unknown :-1 s=CLOSED pgs=16 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:22:33.006 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:32.782+0000 7fc80d35d700 1 --2- 172.21.15.66:0/2341386068 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fc8080f51a0 0x7fc808106f60 unknown :-1 s=CLOSED pgs=102 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:22:33.006 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:32.782+0000 7fc80d35d700 1 -- 172.21.15.66:0/2341386068 >> 172.21.15.66:0/2341386068 conn(0x7fc8080f09a0 msgr2=0x7fc8080f1460 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:22:33.006 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:32.782+0000 7fc80d35d700 1 -- 172.21.15.66:0/2341386068 shutdown_connections 2023-11-17T05:22:33.006 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:32.782+0000 7fc80d35d700 1 -- 172.21.15.66:0/2341386068 wait complete. 2023-11-17T05:22:33.006 INFO:teuthology.orchestra.run.smithi066.stderr:dumped monmap epoch 1 2023-11-17T05:22:33.739 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:22:33 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:22:32.781567+0000 mon.smithi040 (mon.0) 126 : audit [DBG] from='client.? 172.21.15.66:0/2341386068' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-11-17T05:22:34.161 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2023-11-17T05:22:34.161 DEBUG:teuthology.orchestra.run.smithi066:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.4 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ed6470e2-8508-11ee-8dbe-212e2dc638e7 -- ceph mon dump -f json 2023-11-17T05:22:34.739 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:22:34 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:22:33.167753+0000 mgr.smithi040.ogcprl (mgr.14164) 42 : cluster [DBG] pgmap v35: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:22:35.631 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:35.628+0000 7f82f17ed700 1 -- 172.21.15.66:0/2052262581 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f82ec0f47c0 msgr2=0x7f82ec0f4be0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:22:35.631 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:35.628+0000 7f82f17ed700 1 --2- 172.21.15.66:0/2052262581 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f82ec0f47c0 0x7f82ec0f4be0 secure :-1 s=READY pgs=103 cs=0 l=1 rev1=1 rx=0x7f82dc009fb0 tx=0x7f82dc009b30).stop 2023-11-17T05:22:35.631 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:35.628+0000 7f82f17ed700 1 -- 172.21.15.66:0/2052262581 shutdown_connections 2023-11-17T05:22:35.631 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:35.628+0000 7f82f17ed700 1 --2- 172.21.15.66:0/2052262581 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f82ec0f47c0 0x7f82ec0f4be0 unknown :-1 s=CLOSED pgs=103 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:22:35.631 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:35.628+0000 7f82f17ed700 1 -- 172.21.15.66:0/2052262581 >> 172.21.15.66:0/2052262581 conn(0x7f82ec0f02e0 msgr2=0x7f82ec0f2700 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:22:35.631 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:35.628+0000 7f82f17ed700 1 -- 172.21.15.66:0/2052262581 shutdown_connections 2023-11-17T05:22:35.631 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:35.628+0000 7f82f17ed700 1 -- 172.21.15.66:0/2052262581 wait complete. 2023-11-17T05:22:35.631 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:35.629+0000 7f82f17ed700 1 Processor -- start 2023-11-17T05:22:35.632 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:35.629+0000 7f82f17ed700 1 -- start start 2023-11-17T05:22:35.632 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:35.629+0000 7f82f17ed700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f82ec0f47c0 0x7f82ec0f7370 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:22:35.632 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:35.629+0000 7f82f17ed700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f82ec0f57f0 con 0x7f82ec0f47c0 2023-11-17T05:22:35.632 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:35.629+0000 7f82eaffd700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f82ec0f47c0 0x7f82ec0f7370 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:22:35.632 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:35.629+0000 7f82eaffd700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f82ec0f47c0 0x7f82ec0f7370 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.66:36348/0 (socket says 172.21.15.66:36348) 2023-11-17T05:22:35.632 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:35.629+0000 7f82eaffd700 1 -- 172.21.15.66:0/883121824 learned_addr learned my addr 172.21.15.66:0/883121824 (peer_addr_for_me v2:172.21.15.66:0/0) 2023-11-17T05:22:35.632 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:35.630+0000 7f82eaffd700 1 -- 172.21.15.66:0/883121824 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f82dc005040 con 0x7f82ec0f47c0 2023-11-17T05:22:35.632 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:35.630+0000 7f82eaffd700 1 --2- 172.21.15.66:0/883121824 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f82ec0f47c0 0x7f82ec0f7370 secure :-1 s=READY pgs=104 cs=0 l=1 rev1=1 rx=0x7f82dc013510 tx=0x7f82dc00b040).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:22:35.633 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:35.630+0000 7f82e37fe700 1 -- 172.21.15.66:0/883121824 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f82dc009320 con 0x7f82ec0f47c0 2023-11-17T05:22:35.633 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:35.630+0000 7f82f17ed700 1 -- 172.21.15.66:0/883121824 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f82ec0f5990 con 0x7f82ec0f47c0 2023-11-17T05:22:35.634 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:35.630+0000 7f82e37fe700 1 -- 172.21.15.66:0/883121824 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(25 keys) v1 ==== 1074+0+0 (secure 0 0 0) 0x7f82dc013f00 con 0x7f82ec0f47c0 2023-11-17T05:22:35.634 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:35.630+0000 7f82e37fe700 1 -- 172.21.15.66:0/883121824 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f82dc014d90 con 0x7f82ec0f47c0 2023-11-17T05:22:35.634 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:35.630+0000 7f82f17ed700 1 -- 172.21.15.66:0/883121824 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f82ec0f5e20 con 0x7f82ec0f47c0 2023-11-17T05:22:35.634 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:35.631+0000 7f82e37fe700 1 -- 172.21.15.66:0/883121824 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 17) v1 ==== 36269+0+0 (secure 0 0 0) 0x7f82dc014420 con 0x7f82ec0f47c0 2023-11-17T05:22:35.634 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:35.631+0000 7f82f17ed700 1 -- 172.21.15.66:0/883121824 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f82ec059100 con 0x7f82ec0f47c0 2023-11-17T05:22:35.634 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:35.631+0000 7f82e37fe700 1 --2- 172.21.15.66:0/883121824 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f82d802ff60 0x7f82d8032420 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:22:35.634 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:35.631+0000 7f82e37fe700 1 -- 172.21.15.66:0/883121824 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(5..5 src has 1..5) v4 ==== 1270+0+0 (secure 0 0 0) 0x7f82dc040fb0 con 0x7f82ec0f47c0 2023-11-17T05:22:35.634 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:35.631+0000 7f82ea7fc700 1 --2- 172.21.15.66:0/883121824 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f82d802ff60 0x7f82d8032420 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:22:35.634 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:35.632+0000 7f82ea7fc700 1 --2- 172.21.15.66:0/883121824 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f82d802ff60 0x7f82d8032420 secure :-1 s=READY pgs=17 cs=0 l=1 rev1=1 rx=0x7f82d40096a0 tx=0x7f82d4006b40).ready entity=mgr.14164 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:22:35.637 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:35.636+0000 7f82e37fe700 1 -- 172.21.15.66:0/883121824 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+152212 (secure 0 0 0) 0x7f82dc0095f0 con 0x7f82ec0f47c0 2023-11-17T05:22:35.858 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:35.856+0000 7f82f17ed700 1 -- 172.21.15.66:0/883121824 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "mon dump", "format": "json"} v 0) v1 -- 0x7f82ec0fbf60 con 0x7f82ec0f47c0 2023-11-17T05:22:35.858 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:35.857+0000 7f82e37fe700 1 -- 172.21.15.66:0/883121824 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mon_command_ack([{"prefix": "mon dump", "format": "json"}]=0 dumped monmap epoch 1 v1) v1 ==== 95+0+690 (secure 0 0 0) 0x7f82dc01db70 con 0x7f82ec0f47c0 2023-11-17T05:22:35.858 INFO:teuthology.orchestra.run.smithi066.stdout: 2023-11-17T05:22:35.859 INFO:teuthology.orchestra.run.smithi066.stdout:{"epoch":1,"fsid":"ed6470e2-8508-11ee-8dbe-212e2dc638e7","modified":"2023-11-17T05:20:13.934038Z","created":"2023-11-17T05:20:13.934038Z","min_mon_release":16,"min_mon_release_name":"pacific","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging"],"optional":[]},"mons":[{"rank":0,"name":"smithi040","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:3300","nonce":0},{"type":"v1","addr":"172.21.15.40:6789","nonce":0}]},"addr":"172.21.15.40:6789/0","public_addr":"172.21.15.40:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2023-11-17T05:22:35.860 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:35.858+0000 7f82f17ed700 1 -- 172.21.15.66:0/883121824 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f82d802ff60 msgr2=0x7f82d8032420 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:22:35.860 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:35.858+0000 7f82f17ed700 1 --2- 172.21.15.66:0/883121824 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f82d802ff60 0x7f82d8032420 secure :-1 s=READY pgs=17 cs=0 l=1 rev1=1 rx=0x7f82d40096a0 tx=0x7f82d4006b40).stop 2023-11-17T05:22:35.860 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:35.858+0000 7f82f17ed700 1 -- 172.21.15.66:0/883121824 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f82ec0f47c0 msgr2=0x7f82ec0f7370 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:22:35.860 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:35.858+0000 7f82f17ed700 1 --2- 172.21.15.66:0/883121824 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f82ec0f47c0 0x7f82ec0f7370 secure :-1 s=READY pgs=104 cs=0 l=1 rev1=1 rx=0x7f82dc013510 tx=0x7f82dc00b040).stop 2023-11-17T05:22:35.860 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:35.858+0000 7f82f17ed700 1 -- 172.21.15.66:0/883121824 shutdown_connections 2023-11-17T05:22:35.860 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:35.858+0000 7f82f17ed700 1 --2- 172.21.15.66:0/883121824 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f82d802ff60 0x7f82d8032420 unknown :-1 s=CLOSED pgs=17 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:22:35.860 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:35.858+0000 7f82f17ed700 1 --2- 172.21.15.66:0/883121824 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f82ec0f47c0 0x7f82ec0f7370 unknown :-1 s=CLOSED pgs=104 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:22:35.860 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:35.858+0000 7f82f17ed700 1 -- 172.21.15.66:0/883121824 >> 172.21.15.66:0/883121824 conn(0x7f82ec0f02e0 msgr2=0x7f82ec0f0db0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:22:35.860 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:35.858+0000 7f82f17ed700 1 -- 172.21.15.66:0/883121824 shutdown_connections 2023-11-17T05:22:35.861 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:35.858+0000 7f82f17ed700 1 -- 172.21.15.66:0/883121824 wait complete. 2023-11-17T05:22:35.861 INFO:teuthology.orchestra.run.smithi066.stderr:dumped monmap epoch 1 2023-11-17T05:22:36.739 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:22:36 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:22:35.168013+0000 mgr.smithi040.ogcprl (mgr.14164) 43 : cluster [DBG] pgmap v36: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:22:36.739 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:22:36 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:22:35.858212+0000 mon.smithi040 (mon.0) 127 : audit [DBG] from='client.? 172.21.15.66:0/883121824' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-11-17T05:22:37.257 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2023-11-17T05:22:37.257 DEBUG:teuthology.orchestra.run.smithi066:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.4 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ed6470e2-8508-11ee-8dbe-212e2dc638e7 -- ceph mon dump -f json 2023-11-17T05:22:38.455 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:38.451+0000 7f0c582d9700 1 -- 172.21.15.66:0/1467680670 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f0c500f6510 msgr2=0x7f0c500f6930 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:22:38.455 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:38.451+0000 7f0c582d9700 1 --2- 172.21.15.66:0/1467680670 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f0c500f6510 0x7f0c500f6930 secure :-1 s=READY pgs=105 cs=0 l=1 rev1=1 rx=0x7f0c40009fb0 tx=0x7f0c40009b30).stop 2023-11-17T05:22:38.455 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:38.451+0000 7f0c582d9700 1 -- 172.21.15.66:0/1467680670 shutdown_connections 2023-11-17T05:22:38.456 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:38.451+0000 7f0c582d9700 1 --2- 172.21.15.66:0/1467680670 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f0c500f6510 0x7f0c500f6930 unknown :-1 s=CLOSED pgs=105 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:22:38.456 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:38.451+0000 7f0c582d9700 1 -- 172.21.15.66:0/1467680670 >> 172.21.15.66:0/1467680670 conn(0x7f0c500f1c70 msgr2=0x7f0c500f4090 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:22:38.456 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:38.451+0000 7f0c582d9700 1 -- 172.21.15.66:0/1467680670 shutdown_connections 2023-11-17T05:22:38.456 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:38.451+0000 7f0c582d9700 1 -- 172.21.15.66:0/1467680670 wait complete. 2023-11-17T05:22:38.456 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:38.452+0000 7f0c582d9700 1 Processor -- start 2023-11-17T05:22:38.456 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:38.452+0000 7f0c582d9700 1 -- start start 2023-11-17T05:22:38.456 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:38.452+0000 7f0c582d9700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f0c500f6510 0x7f0c50106730 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:22:38.456 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:38.452+0000 7f0c582d9700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f0c50106c20 con 0x7f0c500f6510 2023-11-17T05:22:38.456 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:38.452+0000 7f0c56075700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f0c500f6510 0x7f0c50106730 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:22:38.456 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:38.452+0000 7f0c56075700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f0c500f6510 0x7f0c50106730 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.66:36362/0 (socket says 172.21.15.66:36362) 2023-11-17T05:22:38.456 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:38.452+0000 7f0c56075700 1 -- 172.21.15.66:0/1637538245 learned_addr learned my addr 172.21.15.66:0/1637538245 (peer_addr_for_me v2:172.21.15.66:0/0) 2023-11-17T05:22:38.457 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:38.452+0000 7f0c56075700 1 -- 172.21.15.66:0/1637538245 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f0c40005040 con 0x7f0c500f6510 2023-11-17T05:22:38.457 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:38.453+0000 7f0c56075700 1 --2- 172.21.15.66:0/1637538245 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f0c500f6510 0x7f0c50106730 secure :-1 s=READY pgs=106 cs=0 l=1 rev1=1 rx=0x7f0c40013510 tx=0x7f0c4000b040).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:22:38.457 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:38.453+0000 7f0c46ffd700 1 -- 172.21.15.66:0/1637538245 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f0c40009320 con 0x7f0c500f6510 2023-11-17T05:22:38.457 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:38.453+0000 7f0c582d9700 1 -- 172.21.15.66:0/1637538245 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f0c50106e20 con 0x7f0c500f6510 2023-11-17T05:22:38.457 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:38.453+0000 7f0c46ffd700 1 -- 172.21.15.66:0/1637538245 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(25 keys) v1 ==== 1074+0+0 (secure 0 0 0) 0x7f0c40013f00 con 0x7f0c500f6510 2023-11-17T05:22:38.457 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:38.453+0000 7f0c46ffd700 1 -- 172.21.15.66:0/1637538245 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f0c40014d90 con 0x7f0c500f6510 2023-11-17T05:22:38.457 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:38.453+0000 7f0c582d9700 1 -- 172.21.15.66:0/1637538245 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f0c501072b0 con 0x7f0c500f6510 2023-11-17T05:22:38.457 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:38.454+0000 7f0c46ffd700 1 -- 172.21.15.66:0/1637538245 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 17) v1 ==== 36269+0+0 (secure 0 0 0) 0x7f0c40014420 con 0x7f0c500f6510 2023-11-17T05:22:38.457 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:38.454+0000 7f0c582d9700 1 -- 172.21.15.66:0/1637538245 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f0c50045ad0 con 0x7f0c500f6510 2023-11-17T05:22:38.457 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:38.454+0000 7f0c46ffd700 1 --2- 172.21.15.66:0/1637538245 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f0c3c02ff60 0x7f0c3c032420 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:22:38.457 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:38.454+0000 7f0c46ffd700 1 -- 172.21.15.66:0/1637538245 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(5..5 src has 1..5) v4 ==== 1270+0+0 (secure 0 0 0) 0x7f0c40041000 con 0x7f0c500f6510 2023-11-17T05:22:38.457 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:38.454+0000 7f0c55874700 1 --2- 172.21.15.66:0/1637538245 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f0c3c02ff60 0x7f0c3c032420 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:22:38.458 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:38.454+0000 7f0c55874700 1 --2- 172.21.15.66:0/1637538245 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f0c3c02ff60 0x7f0c3c032420 secure :-1 s=READY pgs=18 cs=0 l=1 rev1=1 rx=0x7f0c4c0096a0 tx=0x7f0c4c006b40).ready entity=mgr.14164 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:22:38.460 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:38.459+0000 7f0c46ffd700 1 -- 172.21.15.66:0/1637538245 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+152212 (secure 0 0 0) 0x7f0c4001dd50 con 0x7f0c500f6510 2023-11-17T05:22:38.680 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:38.678+0000 7f0c582d9700 1 -- 172.21.15.66:0/1637538245 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "mon dump", "format": "json"} v 0) v1 -- 0x7f0c500fbf60 con 0x7f0c500f6510 2023-11-17T05:22:38.681 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:38.679+0000 7f0c46ffd700 1 -- 172.21.15.66:0/1637538245 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mon_command_ack([{"prefix": "mon dump", "format": "json"}]=0 dumped monmap epoch 1 v1) v1 ==== 95+0+690 (secure 0 0 0) 0x7f0c4001db70 con 0x7f0c500f6510 2023-11-17T05:22:38.681 INFO:teuthology.orchestra.run.smithi066.stdout: 2023-11-17T05:22:38.682 INFO:teuthology.orchestra.run.smithi066.stdout:{"epoch":1,"fsid":"ed6470e2-8508-11ee-8dbe-212e2dc638e7","modified":"2023-11-17T05:20:13.934038Z","created":"2023-11-17T05:20:13.934038Z","min_mon_release":16,"min_mon_release_name":"pacific","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging"],"optional":[]},"mons":[{"rank":0,"name":"smithi040","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:3300","nonce":0},{"type":"v1","addr":"172.21.15.40:6789","nonce":0}]},"addr":"172.21.15.40:6789/0","public_addr":"172.21.15.40:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2023-11-17T05:22:38.683 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:38.681+0000 7f0c582d9700 1 -- 172.21.15.66:0/1637538245 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f0c3c02ff60 msgr2=0x7f0c3c032420 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:22:38.683 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:38.681+0000 7f0c582d9700 1 --2- 172.21.15.66:0/1637538245 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f0c3c02ff60 0x7f0c3c032420 secure :-1 s=READY pgs=18 cs=0 l=1 rev1=1 rx=0x7f0c4c0096a0 tx=0x7f0c4c006b40).stop 2023-11-17T05:22:38.684 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:38.681+0000 7f0c582d9700 1 -- 172.21.15.66:0/1637538245 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f0c500f6510 msgr2=0x7f0c50106730 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:22:38.684 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:38.681+0000 7f0c582d9700 1 --2- 172.21.15.66:0/1637538245 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f0c500f6510 0x7f0c50106730 secure :-1 s=READY pgs=106 cs=0 l=1 rev1=1 rx=0x7f0c40013510 tx=0x7f0c4000b040).stop 2023-11-17T05:22:38.684 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:38.681+0000 7f0c582d9700 1 -- 172.21.15.66:0/1637538245 shutdown_connections 2023-11-17T05:22:38.684 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:38.681+0000 7f0c582d9700 1 --2- 172.21.15.66:0/1637538245 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f0c3c02ff60 0x7f0c3c032420 unknown :-1 s=CLOSED pgs=18 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:22:38.684 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:38.681+0000 7f0c582d9700 1 --2- 172.21.15.66:0/1637538245 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f0c500f6510 0x7f0c50106730 unknown :-1 s=CLOSED pgs=106 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:22:38.684 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:38.681+0000 7f0c582d9700 1 -- 172.21.15.66:0/1637538245 >> 172.21.15.66:0/1637538245 conn(0x7f0c500f1c70 msgr2=0x7f0c500f2760 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:22:38.685 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:38.681+0000 7f0c582d9700 1 -- 172.21.15.66:0/1637538245 shutdown_connections 2023-11-17T05:22:38.685 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:38.681+0000 7f0c582d9700 1 -- 172.21.15.66:0/1637538245 wait complete. 2023-11-17T05:22:38.685 INFO:teuthology.orchestra.run.smithi066.stderr:dumped monmap epoch 1 2023-11-17T05:22:38.739 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:22:38 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:22:37.168398+0000 mgr.smithi040.ogcprl (mgr.14164) 44 : cluster [DBG] pgmap v37: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:22:39.741 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:22:39 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:22:38.680820+0000 mon.smithi040 (mon.0) 128 : audit [DBG] from='client.? 172.21.15.66:0/1637538245' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-11-17T05:22:40.090 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2023-11-17T05:22:40.090 DEBUG:teuthology.orchestra.run.smithi066:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.4 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ed6470e2-8508-11ee-8dbe-212e2dc638e7 -- ceph mon dump -f json 2023-11-17T05:22:40.739 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:22:40 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:22:39.168821+0000 mgr.smithi040.ogcprl (mgr.14164) 45 : cluster [DBG] pgmap v38: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:22:41.437 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:41.436+0000 7f6908616700 1 -- 172.21.15.66:0/2130639604 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f69000f5410 msgr2=0x7f69000f5830 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:22:41.437 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:41.436+0000 7f6908616700 1 --2- 172.21.15.66:0/2130639604 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f69000f5410 0x7f69000f5830 secure :-1 s=READY pgs=107 cs=0 l=1 rev1=1 rx=0x7f68f0009fb0 tx=0x7f68f0009b30).stop 2023-11-17T05:22:41.438 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:41.436+0000 7f6908616700 1 -- 172.21.15.66:0/2130639604 shutdown_connections 2023-11-17T05:22:41.438 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:41.436+0000 7f6908616700 1 --2- 172.21.15.66:0/2130639604 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f69000f5410 0x7f69000f5830 unknown :-1 s=CLOSED pgs=107 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:22:41.438 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:41.436+0000 7f6908616700 1 -- 172.21.15.66:0/2130639604 >> 172.21.15.66:0/2130639604 conn(0x7f69000f0b70 msgr2=0x7f69000f2f90 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:22:41.438 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:41.436+0000 7f6908616700 1 -- 172.21.15.66:0/2130639604 shutdown_connections 2023-11-17T05:22:41.438 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:41.436+0000 7f6908616700 1 -- 172.21.15.66:0/2130639604 wait complete. 2023-11-17T05:22:41.439 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:41.436+0000 7f6908616700 1 Processor -- start 2023-11-17T05:22:41.439 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:41.436+0000 7f6908616700 1 -- start start 2023-11-17T05:22:41.440 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:41.437+0000 7f6908616700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f69000f5410 0x7f6900109450 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:22:41.440 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:41.437+0000 7f6908616700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f6900060220 con 0x7f69000f5410 2023-11-17T05:22:41.440 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:41.437+0000 7f69063b2700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f69000f5410 0x7f6900109450 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:22:41.440 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:41.437+0000 7f69063b2700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f69000f5410 0x7f6900109450 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.66:36382/0 (socket says 172.21.15.66:36382) 2023-11-17T05:22:41.440 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:41.437+0000 7f69063b2700 1 -- 172.21.15.66:0/274530399 learned_addr learned my addr 172.21.15.66:0/274530399 (peer_addr_for_me v2:172.21.15.66:0/0) 2023-11-17T05:22:41.440 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:41.437+0000 7f69063b2700 1 -- 172.21.15.66:0/274530399 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f68f0005040 con 0x7f69000f5410 2023-11-17T05:22:41.440 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:41.437+0000 7f69063b2700 1 --2- 172.21.15.66:0/274530399 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f69000f5410 0x7f6900109450 secure :-1 s=READY pgs=108 cs=0 l=1 rev1=1 rx=0x7f68f0013510 tx=0x7f68f000b040).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:22:41.440 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:41.437+0000 7f68f6ffd700 1 -- 172.21.15.66:0/274530399 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f68f0009320 con 0x7f69000f5410 2023-11-17T05:22:41.440 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:41.438+0000 7f6908616700 1 -- 172.21.15.66:0/274530399 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f6900060420 con 0x7f69000f5410 2023-11-17T05:22:41.440 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:41.438+0000 7f68f6ffd700 1 -- 172.21.15.66:0/274530399 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(25 keys) v1 ==== 1074+0+0 (secure 0 0 0) 0x7f68f0013f00 con 0x7f69000f5410 2023-11-17T05:22:41.440 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:41.438+0000 7f68f6ffd700 1 -- 172.21.15.66:0/274530399 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f68f0014d90 con 0x7f69000f5410 2023-11-17T05:22:41.440 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:41.438+0000 7f6908616700 1 -- 172.21.15.66:0/274530399 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f6900109b00 con 0x7f69000f5410 2023-11-17T05:22:41.441 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:41.438+0000 7f68f6ffd700 1 -- 172.21.15.66:0/274530399 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 17) v1 ==== 36269+0+0 (secure 0 0 0) 0x7f68f00263a0 con 0x7f69000f5410 2023-11-17T05:22:41.441 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:41.439+0000 7f68f6ffd700 1 --2- 172.21.15.66:0/274530399 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f68ec02ff60 0x7f68ec032420 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:22:41.441 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:41.439+0000 7f6908616700 1 -- 172.21.15.66:0/274530399 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f6900059100 con 0x7f69000f5410 2023-11-17T05:22:41.441 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:41.439+0000 7f68f6ffd700 1 -- 172.21.15.66:0/274530399 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(5..5 src has 1..5) v4 ==== 1270+0+0 (secure 0 0 0) 0x7f68f0041120 con 0x7f69000f5410 2023-11-17T05:22:41.442 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:41.439+0000 7f6905bb1700 1 --2- 172.21.15.66:0/274530399 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f68ec02ff60 0x7f68ec032420 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:22:41.442 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:41.439+0000 7f6905bb1700 1 --2- 172.21.15.66:0/274530399 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f68ec02ff60 0x7f68ec032420 secure :-1 s=READY pgs=19 cs=0 l=1 rev1=1 rx=0x7f68fc0096a0 tx=0x7f68fc006b40).ready entity=mgr.14164 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:22:41.448 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:41.445+0000 7f68f6ffd700 1 -- 172.21.15.66:0/274530399 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+152212 (secure 0 0 0) 0x7f68f0012920 con 0x7f69000f5410 2023-11-17T05:22:41.666 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:41.664+0000 7f6908616700 1 -- 172.21.15.66:0/274530399 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "mon dump", "format": "json"} v 0) v1 -- 0x7f6900045ad0 con 0x7f69000f5410 2023-11-17T05:22:41.666 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:41.665+0000 7f68f6ffd700 1 -- 172.21.15.66:0/274530399 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mon_command_ack([{"prefix": "mon dump", "format": "json"}]=0 dumped monmap epoch 1 v1) v1 ==== 95+0+690 (secure 0 0 0) 0x7f68f002b020 con 0x7f69000f5410 2023-11-17T05:22:41.667 INFO:teuthology.orchestra.run.smithi066.stdout: 2023-11-17T05:22:41.667 INFO:teuthology.orchestra.run.smithi066.stdout:{"epoch":1,"fsid":"ed6470e2-8508-11ee-8dbe-212e2dc638e7","modified":"2023-11-17T05:20:13.934038Z","created":"2023-11-17T05:20:13.934038Z","min_mon_release":16,"min_mon_release_name":"pacific","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging"],"optional":[]},"mons":[{"rank":0,"name":"smithi040","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:3300","nonce":0},{"type":"v1","addr":"172.21.15.40:6789","nonce":0}]},"addr":"172.21.15.40:6789/0","public_addr":"172.21.15.40:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2023-11-17T05:22:41.670 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:41.667+0000 7f6908616700 1 -- 172.21.15.66:0/274530399 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f68ec02ff60 msgr2=0x7f68ec032420 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:22:41.670 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:41.667+0000 7f6908616700 1 --2- 172.21.15.66:0/274530399 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f68ec02ff60 0x7f68ec032420 secure :-1 s=READY pgs=19 cs=0 l=1 rev1=1 rx=0x7f68fc0096a0 tx=0x7f68fc006b40).stop 2023-11-17T05:22:41.670 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:41.667+0000 7f6908616700 1 -- 172.21.15.66:0/274530399 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f69000f5410 msgr2=0x7f6900109450 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:22:41.670 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:41.667+0000 7f6908616700 1 --2- 172.21.15.66:0/274530399 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f69000f5410 0x7f6900109450 secure :-1 s=READY pgs=108 cs=0 l=1 rev1=1 rx=0x7f68f0013510 tx=0x7f68f000b040).stop 2023-11-17T05:22:41.670 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:41.667+0000 7f6908616700 1 -- 172.21.15.66:0/274530399 shutdown_connections 2023-11-17T05:22:41.670 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:41.667+0000 7f6908616700 1 --2- 172.21.15.66:0/274530399 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f68ec02ff60 0x7f68ec032420 unknown :-1 s=CLOSED pgs=19 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:22:41.671 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:41.667+0000 7f6908616700 1 --2- 172.21.15.66:0/274530399 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f69000f5410 0x7f6900109450 unknown :-1 s=CLOSED pgs=108 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:22:41.671 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:41.667+0000 7f6908616700 1 -- 172.21.15.66:0/274530399 >> 172.21.15.66:0/274530399 conn(0x7f69000f0b70 msgr2=0x7f69000f1660 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:22:41.671 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:41.667+0000 7f6908616700 1 -- 172.21.15.66:0/274530399 shutdown_connections 2023-11-17T05:22:41.671 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:41.667+0000 7f6908616700 1 -- 172.21.15.66:0/274530399 wait complete. 2023-11-17T05:22:41.671 INFO:teuthology.orchestra.run.smithi066.stderr:dumped monmap epoch 1 2023-11-17T05:22:42.739 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:22:42 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:22:41.169325+0000 mgr.smithi040.ogcprl (mgr.14164) 46 : cluster [DBG] pgmap v39: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:22:42.739 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:22:42 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:22:41.666740+0000 mon.smithi040 (mon.0) 129 : audit [DBG] from='client.? 172.21.15.66:0/274530399' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-11-17T05:22:43.047 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2023-11-17T05:22:43.047 DEBUG:teuthology.orchestra.run.smithi066:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.4 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ed6470e2-8508-11ee-8dbe-212e2dc638e7 -- ceph mon dump -f json 2023-11-17T05:22:44.223 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:44.220+0000 7f6ff1987700 1 -- 172.21.15.66:0/1310192072 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f6fec0f5410 msgr2=0x7f6fec0f5830 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:22:44.223 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:44.220+0000 7f6ff1987700 1 --2- 172.21.15.66:0/1310192072 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f6fec0f5410 0x7f6fec0f5830 secure :-1 s=READY pgs=109 cs=0 l=1 rev1=1 rx=0x7f6fdc009fb0 tx=0x7f6fdc009b30).stop 2023-11-17T05:22:44.224 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:44.220+0000 7f6ff1987700 1 -- 172.21.15.66:0/1310192072 shutdown_connections 2023-11-17T05:22:44.224 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:44.220+0000 7f6ff1987700 1 --2- 172.21.15.66:0/1310192072 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f6fec0f5410 0x7f6fec0f5830 unknown :-1 s=CLOSED pgs=109 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:22:44.224 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:44.220+0000 7f6ff1987700 1 -- 172.21.15.66:0/1310192072 >> 172.21.15.66:0/1310192072 conn(0x7f6fec0f0b70 msgr2=0x7f6fec0f2f90 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:22:44.224 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:44.220+0000 7f6ff1987700 1 -- 172.21.15.66:0/1310192072 shutdown_connections 2023-11-17T05:22:44.224 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:44.220+0000 7f6ff1987700 1 -- 172.21.15.66:0/1310192072 wait complete. 2023-11-17T05:22:44.224 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:44.221+0000 7f6ff1987700 1 Processor -- start 2023-11-17T05:22:44.224 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:44.221+0000 7f6ff1987700 1 -- start start 2023-11-17T05:22:44.224 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:44.221+0000 7f6ff1987700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f6fec0f5410 0x7f6fec108910 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:22:44.224 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:44.221+0000 7f6ff1987700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f6fec108e00 con 0x7f6fec0f5410 2023-11-17T05:22:44.224 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:44.221+0000 7f6feaffd700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f6fec0f5410 0x7f6fec108910 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:22:44.224 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:44.221+0000 7f6feaffd700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f6fec0f5410 0x7f6fec108910 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.66:42324/0 (socket says 172.21.15.66:42324) 2023-11-17T05:22:44.224 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:44.221+0000 7f6feaffd700 1 -- 172.21.15.66:0/4129421148 learned_addr learned my addr 172.21.15.66:0/4129421148 (peer_addr_for_me v2:172.21.15.66:0/0) 2023-11-17T05:22:44.224 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:44.222+0000 7f6feaffd700 1 -- 172.21.15.66:0/4129421148 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f6fdc005040 con 0x7f6fec0f5410 2023-11-17T05:22:44.225 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:44.222+0000 7f6feaffd700 1 --2- 172.21.15.66:0/4129421148 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f6fec0f5410 0x7f6fec108910 secure :-1 s=READY pgs=110 cs=0 l=1 rev1=1 rx=0x7f6fdc013510 tx=0x7f6fdc00b040).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:22:44.225 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:44.222+0000 7f6fe37fe700 1 -- 172.21.15.66:0/4129421148 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f6fdc009320 con 0x7f6fec0f5410 2023-11-17T05:22:44.225 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:44.222+0000 7f6ff1987700 1 -- 172.21.15.66:0/4129421148 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f6fec109000 con 0x7f6fec0f5410 2023-11-17T05:22:44.225 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:44.222+0000 7f6fe37fe700 1 -- 172.21.15.66:0/4129421148 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(25 keys) v1 ==== 1074+0+0 (secure 0 0 0) 0x7f6fdc013f00 con 0x7f6fec0f5410 2023-11-17T05:22:44.225 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:44.222+0000 7f6fe37fe700 1 -- 172.21.15.66:0/4129421148 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f6fdc014d90 con 0x7f6fec0f5410 2023-11-17T05:22:44.225 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:44.222+0000 7f6ff1987700 1 -- 172.21.15.66:0/4129421148 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f6fec109490 con 0x7f6fec0f5410 2023-11-17T05:22:44.226 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:44.223+0000 7f6fe37fe700 1 -- 172.21.15.66:0/4129421148 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 17) v1 ==== 36269+0+0 (secure 0 0 0) 0x7f6fdc0095f0 con 0x7f6fec0f5410 2023-11-17T05:22:44.226 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:44.223+0000 7f6ff1987700 1 -- 172.21.15.66:0/4129421148 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f6fec045ad0 con 0x7f6fec0f5410 2023-11-17T05:22:44.226 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:44.223+0000 7f6fe37fe700 1 --2- 172.21.15.66:0/4129421148 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f6fd802ff60 0x7f6fd8032420 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:22:44.226 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:44.223+0000 7f6fe37fe700 1 -- 172.21.15.66:0/4129421148 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(5..5 src has 1..5) v4 ==== 1270+0+0 (secure 0 0 0) 0x7f6fdc041080 con 0x7f6fec0f5410 2023-11-17T05:22:44.226 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:44.223+0000 7f6fea7fc700 1 --2- 172.21.15.66:0/4129421148 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f6fd802ff60 0x7f6fd8032420 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:22:44.226 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:44.224+0000 7f6fea7fc700 1 --2- 172.21.15.66:0/4129421148 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f6fd802ff60 0x7f6fd8032420 secure :-1 s=READY pgs=20 cs=0 l=1 rev1=1 rx=0x7f6fd40096a0 tx=0x7f6fd4006b40).ready entity=mgr.14164 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:22:44.230 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:44.228+0000 7f6fe37fe700 1 -- 172.21.15.66:0/4129421148 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+152212 (secure 0 0 0) 0x7f6fdc014420 con 0x7f6fec0f5410 2023-11-17T05:22:44.452 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:44.451+0000 7f6ff1987700 1 -- 172.21.15.66:0/4129421148 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "mon dump", "format": "json"} v 0) v1 -- 0x7f6fec0fae60 con 0x7f6fec0f5410 2023-11-17T05:22:44.453 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:44.452+0000 7f6fe37fe700 1 -- 172.21.15.66:0/4129421148 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mon_command_ack([{"prefix": "mon dump", "format": "json"}]=0 dumped monmap epoch 1 v1) v1 ==== 95+0+690 (secure 0 0 0) 0x7f6fdc010230 con 0x7f6fec0f5410 2023-11-17T05:22:44.453 INFO:teuthology.orchestra.run.smithi066.stdout: 2023-11-17T05:22:44.453 INFO:teuthology.orchestra.run.smithi066.stdout:{"epoch":1,"fsid":"ed6470e2-8508-11ee-8dbe-212e2dc638e7","modified":"2023-11-17T05:20:13.934038Z","created":"2023-11-17T05:20:13.934038Z","min_mon_release":16,"min_mon_release_name":"pacific","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging"],"optional":[]},"mons":[{"rank":0,"name":"smithi040","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:3300","nonce":0},{"type":"v1","addr":"172.21.15.40:6789","nonce":0}]},"addr":"172.21.15.40:6789/0","public_addr":"172.21.15.40:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2023-11-17T05:22:44.455 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:44.453+0000 7f6ff1987700 1 -- 172.21.15.66:0/4129421148 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f6fd802ff60 msgr2=0x7f6fd8032420 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:22:44.455 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:44.453+0000 7f6ff1987700 1 --2- 172.21.15.66:0/4129421148 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f6fd802ff60 0x7f6fd8032420 secure :-1 s=READY pgs=20 cs=0 l=1 rev1=1 rx=0x7f6fd40096a0 tx=0x7f6fd4006b40).stop 2023-11-17T05:22:44.455 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:44.453+0000 7f6ff1987700 1 -- 172.21.15.66:0/4129421148 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f6fec0f5410 msgr2=0x7f6fec108910 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:22:44.455 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:44.453+0000 7f6ff1987700 1 --2- 172.21.15.66:0/4129421148 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f6fec0f5410 0x7f6fec108910 secure :-1 s=READY pgs=110 cs=0 l=1 rev1=1 rx=0x7f6fdc013510 tx=0x7f6fdc00b040).stop 2023-11-17T05:22:44.455 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:44.453+0000 7f6ff1987700 1 -- 172.21.15.66:0/4129421148 shutdown_connections 2023-11-17T05:22:44.455 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:44.453+0000 7f6ff1987700 1 --2- 172.21.15.66:0/4129421148 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f6fd802ff60 0x7f6fd8032420 unknown :-1 s=CLOSED pgs=20 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:22:44.455 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:44.453+0000 7f6ff1987700 1 --2- 172.21.15.66:0/4129421148 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f6fec0f5410 0x7f6fec108910 unknown :-1 s=CLOSED pgs=110 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:22:44.455 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:44.453+0000 7f6ff1987700 1 -- 172.21.15.66:0/4129421148 >> 172.21.15.66:0/4129421148 conn(0x7f6fec0f0b70 msgr2=0x7f6fec05fe00 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:22:44.455 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:44.453+0000 7f6ff1987700 1 -- 172.21.15.66:0/4129421148 shutdown_connections 2023-11-17T05:22:44.456 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:44.454+0000 7f6ff1987700 1 -- 172.21.15.66:0/4129421148 wait complete. 2023-11-17T05:22:44.456 INFO:teuthology.orchestra.run.smithi066.stderr:dumped monmap epoch 1 2023-11-17T05:22:44.739 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:22:44 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:22:43.169825+0000 mgr.smithi040.ogcprl (mgr.14164) 47 : cluster [DBG] pgmap v40: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:22:45.739 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:22:45 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:22:44.453202+0000 mon.smithi040 (mon.0) 130 : audit [DBG] from='client.? 172.21.15.66:0/4129421148' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-11-17T05:22:45.833 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2023-11-17T05:22:45.833 DEBUG:teuthology.orchestra.run.smithi066:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.4 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ed6470e2-8508-11ee-8dbe-212e2dc638e7 -- ceph mon dump -f json 2023-11-17T05:22:46.739 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:22:46 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:22:45.170345+0000 mgr.smithi040.ogcprl (mgr.14164) 48 : cluster [DBG] pgmap v41: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:22:46.995 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:46.993+0000 7f0a799b9700 1 -- 172.21.15.66:0/1707801317 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f0a740f5410 msgr2=0x7f0a740f5830 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:22:46.995 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:46.993+0000 7f0a799b9700 1 --2- 172.21.15.66:0/1707801317 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f0a740f5410 0x7f0a740f5830 secure :-1 s=READY pgs=111 cs=0 l=1 rev1=1 rx=0x7f0a5c009fb0 tx=0x7f0a5c009b30).stop 2023-11-17T05:22:46.995 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:46.994+0000 7f0a799b9700 1 -- 172.21.15.66:0/1707801317 shutdown_connections 2023-11-17T05:22:46.995 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:46.994+0000 7f0a799b9700 1 --2- 172.21.15.66:0/1707801317 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f0a740f5410 0x7f0a740f5830 unknown :-1 s=CLOSED pgs=111 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:22:46.996 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:46.994+0000 7f0a799b9700 1 -- 172.21.15.66:0/1707801317 >> 172.21.15.66:0/1707801317 conn(0x7f0a740f0b70 msgr2=0x7f0a740f2f90 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:22:46.996 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:46.994+0000 7f0a799b9700 1 -- 172.21.15.66:0/1707801317 shutdown_connections 2023-11-17T05:22:46.996 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:46.994+0000 7f0a799b9700 1 -- 172.21.15.66:0/1707801317 wait complete. 2023-11-17T05:22:46.997 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:46.994+0000 7f0a799b9700 1 Processor -- start 2023-11-17T05:22:46.997 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:46.994+0000 7f0a799b9700 1 -- start start 2023-11-17T05:22:46.997 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:46.994+0000 7f0a799b9700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f0a740f5410 0x7f0a74109450 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:22:46.997 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:46.995+0000 7f0a799b9700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f0a74060220 con 0x7f0a740f5410 2023-11-17T05:22:46.997 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:46.995+0000 7f0a72ffd700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f0a740f5410 0x7f0a74109450 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:22:46.997 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:46.995+0000 7f0a72ffd700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f0a740f5410 0x7f0a74109450 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.66:42334/0 (socket says 172.21.15.66:42334) 2023-11-17T05:22:46.997 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:46.995+0000 7f0a72ffd700 1 -- 172.21.15.66:0/2961985316 learned_addr learned my addr 172.21.15.66:0/2961985316 (peer_addr_for_me v2:172.21.15.66:0/0) 2023-11-17T05:22:46.997 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:46.995+0000 7f0a72ffd700 1 -- 172.21.15.66:0/2961985316 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f0a5c005040 con 0x7f0a740f5410 2023-11-17T05:22:46.998 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:46.995+0000 7f0a72ffd700 1 --2- 172.21.15.66:0/2961985316 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f0a740f5410 0x7f0a74109450 secure :-1 s=READY pgs=112 cs=0 l=1 rev1=1 rx=0x7f0a5c014000 tx=0x7f0a5c00b040).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:22:46.998 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:46.995+0000 7f0a6b7fe700 1 -- 172.21.15.66:0/2961985316 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f0a5c0095f0 con 0x7f0a740f5410 2023-11-17T05:22:46.998 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:46.995+0000 7f0a799b9700 1 -- 172.21.15.66:0/2961985316 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f0a74060420 con 0x7f0a740f5410 2023-11-17T05:22:46.998 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:46.995+0000 7f0a6b7fe700 1 -- 172.21.15.66:0/2961985316 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(25 keys) v1 ==== 1074+0+0 (secure 0 0 0) 0x7f0a5c007e70 con 0x7f0a740f5410 2023-11-17T05:22:46.998 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:46.995+0000 7f0a6b7fe700 1 -- 172.21.15.66:0/2961985316 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f0a5c014ea0 con 0x7f0a740f5410 2023-11-17T05:22:46.998 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:46.995+0000 7f0a799b9700 1 -- 172.21.15.66:0/2961985316 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f0a74109b00 con 0x7f0a740f5410 2023-11-17T05:22:46.999 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:46.996+0000 7f0a6b7fe700 1 -- 172.21.15.66:0/2961985316 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 17) v1 ==== 36269+0+0 (secure 0 0 0) 0x7f0a5c025070 con 0x7f0a740f5410 2023-11-17T05:22:46.999 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:46.996+0000 7f0a799b9700 1 -- 172.21.15.66:0/2961985316 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f0a74059100 con 0x7f0a740f5410 2023-11-17T05:22:46.999 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:46.996+0000 7f0a6b7fe700 1 --2- 172.21.15.66:0/2961985316 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f0a6002ff60 0x7f0a60032420 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:22:46.999 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:46.996+0000 7f0a6b7fe700 1 -- 172.21.15.66:0/2961985316 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(5..5 src has 1..5) v4 ==== 1270+0+0 (secure 0 0 0) 0x7f0a5c040a10 con 0x7f0a740f5410 2023-11-17T05:22:46.999 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:46.997+0000 7f0a727fc700 1 --2- 172.21.15.66:0/2961985316 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f0a6002ff60 0x7f0a60032420 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:22:46.999 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:46.997+0000 7f0a727fc700 1 --2- 172.21.15.66:0/2961985316 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f0a6002ff60 0x7f0a60032420 secure :-1 s=READY pgs=21 cs=0 l=1 rev1=1 rx=0x7f0a640096a0 tx=0x7f0a64006b40).ready entity=mgr.14164 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:22:47.002 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:47.001+0000 7f0a6b7fe700 1 -- 172.21.15.66:0/2961985316 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+152212 (secure 0 0 0) 0x7f0a5c0079c0 con 0x7f0a740f5410 2023-11-17T05:22:47.223 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:47.222+0000 7f0a799b9700 1 -- 172.21.15.66:0/2961985316 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "mon dump", "format": "json"} v 0) v1 -- 0x7f0a74045ad0 con 0x7f0a740f5410 2023-11-17T05:22:47.224 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:47.222+0000 7f0a6b7fe700 1 -- 172.21.15.66:0/2961985316 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mon_command_ack([{"prefix": "mon dump", "format": "json"}]=0 dumped monmap epoch 1 v1) v1 ==== 95+0+690 (secure 0 0 0) 0x7f0a5c00f300 con 0x7f0a740f5410 2023-11-17T05:22:47.224 INFO:teuthology.orchestra.run.smithi066.stdout: 2023-11-17T05:22:47.224 INFO:teuthology.orchestra.run.smithi066.stdout:{"epoch":1,"fsid":"ed6470e2-8508-11ee-8dbe-212e2dc638e7","modified":"2023-11-17T05:20:13.934038Z","created":"2023-11-17T05:20:13.934038Z","min_mon_release":16,"min_mon_release_name":"pacific","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging"],"optional":[]},"mons":[{"rank":0,"name":"smithi040","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:3300","nonce":0},{"type":"v1","addr":"172.21.15.40:6789","nonce":0}]},"addr":"172.21.15.40:6789/0","public_addr":"172.21.15.40:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2023-11-17T05:22:47.226 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:47.224+0000 7f0a799b9700 1 -- 172.21.15.66:0/2961985316 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f0a6002ff60 msgr2=0x7f0a60032420 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:22:47.226 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:47.224+0000 7f0a799b9700 1 --2- 172.21.15.66:0/2961985316 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f0a6002ff60 0x7f0a60032420 secure :-1 s=READY pgs=21 cs=0 l=1 rev1=1 rx=0x7f0a640096a0 tx=0x7f0a64006b40).stop 2023-11-17T05:22:47.226 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:47.224+0000 7f0a799b9700 1 -- 172.21.15.66:0/2961985316 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f0a740f5410 msgr2=0x7f0a74109450 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:22:47.226 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:47.224+0000 7f0a799b9700 1 --2- 172.21.15.66:0/2961985316 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f0a740f5410 0x7f0a74109450 secure :-1 s=READY pgs=112 cs=0 l=1 rev1=1 rx=0x7f0a5c014000 tx=0x7f0a5c00b040).stop 2023-11-17T05:22:47.226 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:47.224+0000 7f0a799b9700 1 -- 172.21.15.66:0/2961985316 shutdown_connections 2023-11-17T05:22:47.226 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:47.224+0000 7f0a799b9700 1 --2- 172.21.15.66:0/2961985316 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f0a6002ff60 0x7f0a60032420 unknown :-1 s=CLOSED pgs=21 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:22:47.226 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:47.224+0000 7f0a799b9700 1 --2- 172.21.15.66:0/2961985316 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f0a740f5410 0x7f0a74109450 unknown :-1 s=CLOSED pgs=112 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:22:47.226 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:47.224+0000 7f0a799b9700 1 -- 172.21.15.66:0/2961985316 >> 172.21.15.66:0/2961985316 conn(0x7f0a740f0b70 msgr2=0x7f0a740f1660 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:22:47.226 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:47.224+0000 7f0a799b9700 1 -- 172.21.15.66:0/2961985316 shutdown_connections 2023-11-17T05:22:47.226 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:47.224+0000 7f0a799b9700 1 -- 172.21.15.66:0/2961985316 wait complete. 2023-11-17T05:22:47.227 INFO:teuthology.orchestra.run.smithi066.stderr:dumped monmap epoch 1 2023-11-17T05:22:47.739 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:22:47 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:22:47.223957+0000 mon.smithi040 (mon.0) 131 : audit [DBG] from='client.? 172.21.15.66:0/2961985316' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-11-17T05:22:48.617 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2023-11-17T05:22:48.617 DEBUG:teuthology.orchestra.run.smithi066:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.4 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ed6470e2-8508-11ee-8dbe-212e2dc638e7 -- ceph mon dump -f json 2023-11-17T05:22:48.738 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:22:48 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:22:47.170724+0000 mgr.smithi040.ogcprl (mgr.14164) 49 : cluster [DBG] pgmap v42: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:22:49.820 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:49.817+0000 7f044fd27700 1 -- 172.21.15.66:0/536461542 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f04480f6510 msgr2=0x7f04480f6930 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:22:49.820 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:49.817+0000 7f044fd27700 1 --2- 172.21.15.66:0/536461542 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f04480f6510 0x7f04480f6930 secure :-1 s=READY pgs=113 cs=0 l=1 rev1=1 rx=0x7f0444009fb0 tx=0x7f0444009b30).stop 2023-11-17T05:22:49.820 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:49.817+0000 7f044fd27700 1 -- 172.21.15.66:0/536461542 shutdown_connections 2023-11-17T05:22:49.820 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:49.817+0000 7f044fd27700 1 --2- 172.21.15.66:0/536461542 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f04480f6510 0x7f04480f6930 unknown :-1 s=CLOSED pgs=113 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:22:49.820 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:49.817+0000 7f044fd27700 1 -- 172.21.15.66:0/536461542 >> 172.21.15.66:0/536461542 conn(0x7f04480f1c70 msgr2=0x7f04480f4090 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:22:49.820 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:49.817+0000 7f044fd27700 1 -- 172.21.15.66:0/536461542 shutdown_connections 2023-11-17T05:22:49.820 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:49.817+0000 7f044fd27700 1 -- 172.21.15.66:0/536461542 wait complete. 2023-11-17T05:22:49.820 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:49.817+0000 7f044fd27700 1 Processor -- start 2023-11-17T05:22:49.820 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:49.817+0000 7f044fd27700 1 -- start start 2023-11-17T05:22:49.820 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:49.817+0000 7f044fd27700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f04480f6510 0x7f0448106730 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:22:49.820 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:49.817+0000 7f044fd27700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f0448106c20 con 0x7f04480f6510 2023-11-17T05:22:49.821 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:49.818+0000 7f044dac3700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f04480f6510 0x7f0448106730 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:22:49.821 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:49.818+0000 7f044dac3700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f04480f6510 0x7f0448106730 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.66:42350/0 (socket says 172.21.15.66:42350) 2023-11-17T05:22:49.821 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:49.818+0000 7f044dac3700 1 -- 172.21.15.66:0/1100819385 learned_addr learned my addr 172.21.15.66:0/1100819385 (peer_addr_for_me v2:172.21.15.66:0/0) 2023-11-17T05:22:49.821 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:49.818+0000 7f044dac3700 1 -- 172.21.15.66:0/1100819385 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f0444005040 con 0x7f04480f6510 2023-11-17T05:22:49.821 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:49.818+0000 7f044dac3700 1 --2- 172.21.15.66:0/1100819385 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f04480f6510 0x7f0448106730 secure :-1 s=READY pgs=114 cs=0 l=1 rev1=1 rx=0x7f0444013510 tx=0x7f044400b040).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:22:49.821 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:49.818+0000 7f043a7fc700 1 -- 172.21.15.66:0/1100819385 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f0444009320 con 0x7f04480f6510 2023-11-17T05:22:49.821 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:49.818+0000 7f044fd27700 1 -- 172.21.15.66:0/1100819385 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f0448106e20 con 0x7f04480f6510 2023-11-17T05:22:49.821 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:49.818+0000 7f043a7fc700 1 -- 172.21.15.66:0/1100819385 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(25 keys) v1 ==== 1074+0+0 (secure 0 0 0) 0x7f0444013f00 con 0x7f04480f6510 2023-11-17T05:22:49.821 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:49.818+0000 7f043a7fc700 1 -- 172.21.15.66:0/1100819385 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f0444014d90 con 0x7f04480f6510 2023-11-17T05:22:49.821 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:49.818+0000 7f044fd27700 1 -- 172.21.15.66:0/1100819385 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f04481072b0 con 0x7f04480f6510 2023-11-17T05:22:49.822 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:49.819+0000 7f043a7fc700 1 -- 172.21.15.66:0/1100819385 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 17) v1 ==== 36269+0+0 (secure 0 0 0) 0x7f04440095f0 con 0x7f04480f6510 2023-11-17T05:22:49.822 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:49.819+0000 7f044fd27700 1 -- 172.21.15.66:0/1100819385 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f0448045ad0 con 0x7f04480f6510 2023-11-17T05:22:49.822 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:49.819+0000 7f043a7fc700 1 --2- 172.21.15.66:0/1100819385 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f043402ffb0 0x7f0434032470 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:22:49.822 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:49.819+0000 7f043a7fc700 1 -- 172.21.15.66:0/1100819385 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(5..5 src has 1..5) v4 ==== 1270+0+0 (secure 0 0 0) 0x7f0444040f80 con 0x7f04480f6510 2023-11-17T05:22:49.822 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:49.819+0000 7f044d2c2700 1 --2- 172.21.15.66:0/1100819385 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f043402ffb0 0x7f0434032470 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:22:49.822 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:49.820+0000 7f044d2c2700 1 --2- 172.21.15.66:0/1100819385 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f043402ffb0 0x7f0434032470 secure :-1 s=READY pgs=22 cs=0 l=1 rev1=1 rx=0x7f043c0096a0 tx=0x7f043c006b40).ready entity=mgr.14164 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:22:49.825 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:49.824+0000 7f043a7fc700 1 -- 172.21.15.66:0/1100819385 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+152212 (secure 0 0 0) 0x7f0444014420 con 0x7f04480f6510 2023-11-17T05:22:50.046 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:50.045+0000 7f044fd27700 1 -- 172.21.15.66:0/1100819385 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "mon dump", "format": "json"} v 0) v1 -- 0x7f04480fbf60 con 0x7f04480f6510 2023-11-17T05:22:50.047 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:50.046+0000 7f043a7fc700 1 -- 172.21.15.66:0/1100819385 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mon_command_ack([{"prefix": "mon dump", "format": "json"}]=0 dumped monmap epoch 1 v1) v1 ==== 95+0+690 (secure 0 0 0) 0x7f044401d7f0 con 0x7f04480f6510 2023-11-17T05:22:50.047 INFO:teuthology.orchestra.run.smithi066.stdout: 2023-11-17T05:22:50.047 INFO:teuthology.orchestra.run.smithi066.stdout:{"epoch":1,"fsid":"ed6470e2-8508-11ee-8dbe-212e2dc638e7","modified":"2023-11-17T05:20:13.934038Z","created":"2023-11-17T05:20:13.934038Z","min_mon_release":16,"min_mon_release_name":"pacific","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging"],"optional":[]},"mons":[{"rank":0,"name":"smithi040","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:3300","nonce":0},{"type":"v1","addr":"172.21.15.40:6789","nonce":0}]},"addr":"172.21.15.40:6789/0","public_addr":"172.21.15.40:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2023-11-17T05:22:50.049 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:50.047+0000 7f044fd27700 1 -- 172.21.15.66:0/1100819385 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f043402ffb0 msgr2=0x7f0434032470 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:22:50.049 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:50.047+0000 7f044fd27700 1 --2- 172.21.15.66:0/1100819385 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f043402ffb0 0x7f0434032470 secure :-1 s=READY pgs=22 cs=0 l=1 rev1=1 rx=0x7f043c0096a0 tx=0x7f043c006b40).stop 2023-11-17T05:22:50.049 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:50.047+0000 7f044fd27700 1 -- 172.21.15.66:0/1100819385 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f04480f6510 msgr2=0x7f0448106730 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:22:50.049 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:50.047+0000 7f044fd27700 1 --2- 172.21.15.66:0/1100819385 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f04480f6510 0x7f0448106730 secure :-1 s=READY pgs=114 cs=0 l=1 rev1=1 rx=0x7f0444013510 tx=0x7f044400b040).stop 2023-11-17T05:22:50.049 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:50.047+0000 7f044fd27700 1 -- 172.21.15.66:0/1100819385 shutdown_connections 2023-11-17T05:22:50.049 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:50.047+0000 7f044fd27700 1 --2- 172.21.15.66:0/1100819385 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f043402ffb0 0x7f0434032470 unknown :-1 s=CLOSED pgs=22 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:22:50.049 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:50.047+0000 7f044fd27700 1 --2- 172.21.15.66:0/1100819385 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f04480f6510 0x7f0448106730 unknown :-1 s=CLOSED pgs=114 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:22:50.049 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:50.047+0000 7f044fd27700 1 -- 172.21.15.66:0/1100819385 >> 172.21.15.66:0/1100819385 conn(0x7f04480f1c70 msgr2=0x7f04480f2760 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:22:50.050 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:50.047+0000 7f044fd27700 1 -- 172.21.15.66:0/1100819385 shutdown_connections 2023-11-17T05:22:50.050 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:50.047+0000 7f044fd27700 1 -- 172.21.15.66:0/1100819385 wait complete. 2023-11-17T05:22:50.050 INFO:teuthology.orchestra.run.smithi066.stderr:dumped monmap epoch 1 2023-11-17T05:22:50.739 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:22:50 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:22:49.171087+0000 mgr.smithi040.ogcprl (mgr.14164) 50 : cluster [DBG] pgmap v43: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:22:50.739 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:22:50 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:22:50.047242+0000 mon.smithi040 (mon.0) 132 : audit [DBG] from='client.? 172.21.15.66:0/1100819385' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-11-17T05:22:51.478 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2023-11-17T05:22:51.479 DEBUG:teuthology.orchestra.run.smithi066:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.4 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ed6470e2-8508-11ee-8dbe-212e2dc638e7 -- ceph mon dump -f json 2023-11-17T05:22:52.637 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:52.635+0000 7fd701ddb700 1 -- 172.21.15.66:0/3872997644 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fd6fc0f5410 msgr2=0x7fd6fc0f5830 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:22:52.637 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:52.635+0000 7fd701ddb700 1 --2- 172.21.15.66:0/3872997644 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fd6fc0f5410 0x7fd6fc0f5830 secure :-1 s=READY pgs=115 cs=0 l=1 rev1=1 rx=0x7fd6ec009fb0 tx=0x7fd6ec009b30).stop 2023-11-17T05:22:52.637 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:52.635+0000 7fd701ddb700 1 -- 172.21.15.66:0/3872997644 shutdown_connections 2023-11-17T05:22:52.637 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:52.635+0000 7fd701ddb700 1 --2- 172.21.15.66:0/3872997644 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fd6fc0f5410 0x7fd6fc0f5830 unknown :-1 s=CLOSED pgs=115 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:22:52.638 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:52.635+0000 7fd701ddb700 1 -- 172.21.15.66:0/3872997644 >> 172.21.15.66:0/3872997644 conn(0x7fd6fc0f0b70 msgr2=0x7fd6fc0f2f90 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:22:52.638 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:52.635+0000 7fd701ddb700 1 -- 172.21.15.66:0/3872997644 shutdown_connections 2023-11-17T05:22:52.638 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:52.635+0000 7fd701ddb700 1 -- 172.21.15.66:0/3872997644 wait complete. 2023-11-17T05:22:52.638 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:52.636+0000 7fd701ddb700 1 Processor -- start 2023-11-17T05:22:52.642 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:52.636+0000 7fd701ddb700 1 -- start start 2023-11-17T05:22:52.642 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:52.636+0000 7fd701ddb700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fd6fc0f5410 0x7fd6fc105630 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:22:52.642 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:52.636+0000 7fd701ddb700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fd6fc105b20 con 0x7fd6fc0f5410 2023-11-17T05:22:52.642 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:52.636+0000 7fd6fb7fe700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fd6fc0f5410 0x7fd6fc105630 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:22:52.642 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:52.637+0000 7fd6fb7fe700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fd6fc0f5410 0x7fd6fc105630 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.66:33252/0 (socket says 172.21.15.66:33252) 2023-11-17T05:22:52.642 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:52.637+0000 7fd6fb7fe700 1 -- 172.21.15.66:0/1214486711 learned_addr learned my addr 172.21.15.66:0/1214486711 (peer_addr_for_me v2:172.21.15.66:0/0) 2023-11-17T05:22:52.643 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:52.637+0000 7fd6fb7fe700 1 -- 172.21.15.66:0/1214486711 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fd6ec005040 con 0x7fd6fc0f5410 2023-11-17T05:22:52.643 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:52.637+0000 7fd6fb7fe700 1 --2- 172.21.15.66:0/1214486711 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fd6fc0f5410 0x7fd6fc105630 secure :-1 s=READY pgs=116 cs=0 l=1 rev1=1 rx=0x7fd6ec013510 tx=0x7fd6ec00b040).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:22:52.643 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:52.637+0000 7fd6f3fff700 1 -- 172.21.15.66:0/1214486711 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7fd6ec009320 con 0x7fd6fc0f5410 2023-11-17T05:22:52.643 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:52.637+0000 7fd701ddb700 1 -- 172.21.15.66:0/1214486711 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fd6fc105d20 con 0x7fd6fc0f5410 2023-11-17T05:22:52.643 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:52.637+0000 7fd6f3fff700 1 -- 172.21.15.66:0/1214486711 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(25 keys) v1 ==== 1074+0+0 (secure 0 0 0) 0x7fd6ec013f00 con 0x7fd6fc0f5410 2023-11-17T05:22:52.643 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:52.637+0000 7fd6f3fff700 1 -- 172.21.15.66:0/1214486711 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7fd6ec014d90 con 0x7fd6fc0f5410 2023-11-17T05:22:52.643 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:52.637+0000 7fd701ddb700 1 -- 172.21.15.66:0/1214486711 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fd6fc1061b0 con 0x7fd6fc0f5410 2023-11-17T05:22:52.643 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:52.638+0000 7fd6f3fff700 1 -- 172.21.15.66:0/1214486711 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 17) v1 ==== 36269+0+0 (secure 0 0 0) 0x7fd6ec014420 con 0x7fd6fc0f5410 2023-11-17T05:22:52.643 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:52.638+0000 7fd701ddb700 1 -- 172.21.15.66:0/1214486711 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fd6fc045ad0 con 0x7fd6fc0f5410 2023-11-17T05:22:52.643 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:52.638+0000 7fd6f3fff700 1 --2- 172.21.15.66:0/1214486711 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fd6e802ff60 0x7fd6e8032420 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:22:52.643 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:52.638+0000 7fd6f3fff700 1 -- 172.21.15.66:0/1214486711 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(5..5 src has 1..5) v4 ==== 1270+0+0 (secure 0 0 0) 0x7fd6ec041070 con 0x7fd6fc0f5410 2023-11-17T05:22:52.643 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:52.638+0000 7fd6faffd700 1 --2- 172.21.15.66:0/1214486711 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fd6e802ff60 0x7fd6e8032420 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:22:52.643 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:52.639+0000 7fd6faffd700 1 --2- 172.21.15.66:0/1214486711 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fd6e802ff60 0x7fd6e8032420 secure :-1 s=READY pgs=23 cs=0 l=1 rev1=1 rx=0x7fd6e40096a0 tx=0x7fd6e4006b40).ready entity=mgr.14164 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:22:52.646 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:52.643+0000 7fd6f3fff700 1 -- 172.21.15.66:0/1214486711 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+152212 (secure 0 0 0) 0x7fd6ec0095f0 con 0x7fd6fc0f5410 2023-11-17T05:22:52.739 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:22:52 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:22:51.171440+0000 mgr.smithi040.ogcprl (mgr.14164) 51 : cluster [DBG] pgmap v44: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:22:52.865 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:52.864+0000 7fd701ddb700 1 -- 172.21.15.66:0/1214486711 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "mon dump", "format": "json"} v 0) v1 -- 0x7fd6fc0fae60 con 0x7fd6fc0f5410 2023-11-17T05:22:52.866 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:52.865+0000 7fd6f3fff700 1 -- 172.21.15.66:0/1214486711 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mon_command_ack([{"prefix": "mon dump", "format": "json"}]=0 dumped monmap epoch 1 v1) v1 ==== 95+0+690 (secure 0 0 0) 0x7fd6ec01db70 con 0x7fd6fc0f5410 2023-11-17T05:22:52.866 INFO:teuthology.orchestra.run.smithi066.stdout: 2023-11-17T05:22:52.867 INFO:teuthology.orchestra.run.smithi066.stdout:{"epoch":1,"fsid":"ed6470e2-8508-11ee-8dbe-212e2dc638e7","modified":"2023-11-17T05:20:13.934038Z","created":"2023-11-17T05:20:13.934038Z","min_mon_release":16,"min_mon_release_name":"pacific","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging"],"optional":[]},"mons":[{"rank":0,"name":"smithi040","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:3300","nonce":0},{"type":"v1","addr":"172.21.15.40:6789","nonce":0}]},"addr":"172.21.15.40:6789/0","public_addr":"172.21.15.40:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2023-11-17T05:22:52.870 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:52.866+0000 7fd701ddb700 1 -- 172.21.15.66:0/1214486711 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fd6e802ff60 msgr2=0x7fd6e8032420 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:22:52.870 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:52.866+0000 7fd701ddb700 1 --2- 172.21.15.66:0/1214486711 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fd6e802ff60 0x7fd6e8032420 secure :-1 s=READY pgs=23 cs=0 l=1 rev1=1 rx=0x7fd6e40096a0 tx=0x7fd6e4006b40).stop 2023-11-17T05:22:52.871 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:52.866+0000 7fd701ddb700 1 -- 172.21.15.66:0/1214486711 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fd6fc0f5410 msgr2=0x7fd6fc105630 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:22:52.871 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:52.866+0000 7fd701ddb700 1 --2- 172.21.15.66:0/1214486711 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fd6fc0f5410 0x7fd6fc105630 secure :-1 s=READY pgs=116 cs=0 l=1 rev1=1 rx=0x7fd6ec013510 tx=0x7fd6ec00b040).stop 2023-11-17T05:22:52.871 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:52.867+0000 7fd701ddb700 1 -- 172.21.15.66:0/1214486711 shutdown_connections 2023-11-17T05:22:52.871 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:52.867+0000 7fd701ddb700 1 --2- 172.21.15.66:0/1214486711 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fd6e802ff60 0x7fd6e8032420 unknown :-1 s=CLOSED pgs=23 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:22:52.871 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:52.867+0000 7fd701ddb700 1 --2- 172.21.15.66:0/1214486711 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fd6fc0f5410 0x7fd6fc105630 unknown :-1 s=CLOSED pgs=116 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:22:52.872 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:52.867+0000 7fd701ddb700 1 -- 172.21.15.66:0/1214486711 >> 172.21.15.66:0/1214486711 conn(0x7fd6fc0f0b70 msgr2=0x7fd6fc0f1660 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:22:52.872 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:52.867+0000 7fd701ddb700 1 -- 172.21.15.66:0/1214486711 shutdown_connections 2023-11-17T05:22:52.872 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:52.867+0000 7fd701ddb700 1 -- 172.21.15.66:0/1214486711 wait complete. 2023-11-17T05:22:52.872 INFO:teuthology.orchestra.run.smithi066.stderr:dumped monmap epoch 1 2023-11-17T05:22:53.739 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:22:53 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:22:52.866509+0000 mon.smithi040 (mon.0) 133 : audit [DBG] from='client.? 172.21.15.66:0/1214486711' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-11-17T05:22:54.258 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2023-11-17T05:22:54.259 DEBUG:teuthology.orchestra.run.smithi066:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.4 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ed6470e2-8508-11ee-8dbe-212e2dc638e7 -- ceph mon dump -f json 2023-11-17T05:22:54.739 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:22:54 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:22:53.171812+0000 mgr.smithi040.ogcprl (mgr.14164) 52 : cluster [DBG] pgmap v45: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:22:55.404 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:55.401+0000 7fb92ffbe700 1 -- 172.21.15.66:0/1721859880 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fb928060280 msgr2=0x7fb9280f90f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:22:55.405 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:55.401+0000 7fb92ffbe700 1 --2- 172.21.15.66:0/1721859880 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fb928060280 0x7fb9280f90f0 secure :-1 s=READY pgs=117 cs=0 l=1 rev1=1 rx=0x7fb924009fb0 tx=0x7fb924009b30).stop 2023-11-17T05:22:55.405 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:55.401+0000 7fb92ffbe700 1 -- 172.21.15.66:0/1721859880 shutdown_connections 2023-11-17T05:22:55.405 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:55.401+0000 7fb92ffbe700 1 --2- 172.21.15.66:0/1721859880 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fb928060280 0x7fb9280f90f0 unknown :-1 s=CLOSED pgs=117 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:22:55.405 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:55.401+0000 7fb92ffbe700 1 -- 172.21.15.66:0/1721859880 >> 172.21.15.66:0/1721859880 conn(0x7fb9280f0b70 msgr2=0x7fb9280f2f90 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:22:55.405 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:55.402+0000 7fb92ffbe700 1 -- 172.21.15.66:0/1721859880 shutdown_connections 2023-11-17T05:22:55.405 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:55.402+0000 7fb92ffbe700 1 -- 172.21.15.66:0/1721859880 wait complete. 2023-11-17T05:22:55.405 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:55.402+0000 7fb92ffbe700 1 Processor -- start 2023-11-17T05:22:55.405 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:55.402+0000 7fb92ffbe700 1 -- start start 2023-11-17T05:22:55.405 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:55.402+0000 7fb92ffbe700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fb928060280 0x7fb9281066e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:22:55.405 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:55.402+0000 7fb92ffbe700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fb928106bd0 con 0x7fb928060280 2023-11-17T05:22:55.405 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:55.402+0000 7fb92dd5a700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fb928060280 0x7fb9281066e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:22:55.405 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:55.403+0000 7fb92dd5a700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fb928060280 0x7fb9281066e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.66:33274/0 (socket says 172.21.15.66:33274) 2023-11-17T05:22:55.405 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:55.403+0000 7fb92dd5a700 1 -- 172.21.15.66:0/4012507317 learned_addr learned my addr 172.21.15.66:0/4012507317 (peer_addr_for_me v2:172.21.15.66:0/0) 2023-11-17T05:22:55.406 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:55.403+0000 7fb92dd5a700 1 -- 172.21.15.66:0/4012507317 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fb924005040 con 0x7fb928060280 2023-11-17T05:22:55.406 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:55.403+0000 7fb92dd5a700 1 --2- 172.21.15.66:0/4012507317 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fb928060280 0x7fb9281066e0 secure :-1 s=READY pgs=118 cs=0 l=1 rev1=1 rx=0x7fb924013510 tx=0x7fb92400b040).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:22:55.406 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:55.403+0000 7fb91a7fc700 1 -- 172.21.15.66:0/4012507317 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7fb924009320 con 0x7fb928060280 2023-11-17T05:22:55.406 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:55.403+0000 7fb92ffbe700 1 -- 172.21.15.66:0/4012507317 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fb928106dd0 con 0x7fb928060280 2023-11-17T05:22:55.406 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:55.403+0000 7fb91a7fc700 1 -- 172.21.15.66:0/4012507317 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(25 keys) v1 ==== 1074+0+0 (secure 0 0 0) 0x7fb924013f00 con 0x7fb928060280 2023-11-17T05:22:55.406 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:55.403+0000 7fb91a7fc700 1 -- 172.21.15.66:0/4012507317 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7fb924014dc0 con 0x7fb928060280 2023-11-17T05:22:55.406 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:55.403+0000 7fb92ffbe700 1 -- 172.21.15.66:0/4012507317 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fb928107260 con 0x7fb928060280 2023-11-17T05:22:55.407 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:55.404+0000 7fb91a7fc700 1 -- 172.21.15.66:0/4012507317 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 17) v1 ==== 36269+0+0 (secure 0 0 0) 0x7fb92401d3d0 con 0x7fb928060280 2023-11-17T05:22:55.407 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:55.404+0000 7fb91a7fc700 1 --2- 172.21.15.66:0/4012507317 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fb91402ff60 0x7fb914032420 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:22:55.407 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:55.404+0000 7fb91a7fc700 1 -- 172.21.15.66:0/4012507317 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(5..5 src has 1..5) v4 ==== 1270+0+0 (secure 0 0 0) 0x7fb924041040 con 0x7fb928060280 2023-11-17T05:22:55.407 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:55.404+0000 7fb92ffbe700 1 -- 172.21.15.66:0/4012507317 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fb928045ad0 con 0x7fb928060280 2023-11-17T05:22:55.407 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:55.404+0000 7fb92d559700 1 --2- 172.21.15.66:0/4012507317 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fb91402ff60 0x7fb914032420 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:22:55.408 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:55.405+0000 7fb92d559700 1 --2- 172.21.15.66:0/4012507317 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fb91402ff60 0x7fb914032420 secure :-1 s=READY pgs=24 cs=0 l=1 rev1=1 rx=0x7fb91c0096a0 tx=0x7fb91c006b40).ready entity=mgr.14164 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:22:55.410 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:55.409+0000 7fb91a7fc700 1 -- 172.21.15.66:0/4012507317 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+152212 (secure 0 0 0) 0x7fb924012920 con 0x7fb928060280 2023-11-17T05:22:55.630 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:55.629+0000 7fb92ffbe700 1 -- 172.21.15.66:0/4012507317 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "mon dump", "format": "json"} v 0) v1 -- 0x7fb928108110 con 0x7fb928060280 2023-11-17T05:22:55.631 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:55.630+0000 7fb91a7fc700 1 -- 172.21.15.66:0/4012507317 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mon_command_ack([{"prefix": "mon dump", "format": "json"}]=0 dumped monmap epoch 1 v1) v1 ==== 95+0+690 (secure 0 0 0) 0x7fb92402b020 con 0x7fb928060280 2023-11-17T05:22:55.631 INFO:teuthology.orchestra.run.smithi066.stdout: 2023-11-17T05:22:55.631 INFO:teuthology.orchestra.run.smithi066.stdout:{"epoch":1,"fsid":"ed6470e2-8508-11ee-8dbe-212e2dc638e7","modified":"2023-11-17T05:20:13.934038Z","created":"2023-11-17T05:20:13.934038Z","min_mon_release":16,"min_mon_release_name":"pacific","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging"],"optional":[]},"mons":[{"rank":0,"name":"smithi040","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:3300","nonce":0},{"type":"v1","addr":"172.21.15.40:6789","nonce":0}]},"addr":"172.21.15.40:6789/0","public_addr":"172.21.15.40:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2023-11-17T05:22:55.633 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:55.631+0000 7fb92ffbe700 1 -- 172.21.15.66:0/4012507317 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fb91402ff60 msgr2=0x7fb914032420 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:22:55.633 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:55.631+0000 7fb92ffbe700 1 --2- 172.21.15.66:0/4012507317 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fb91402ff60 0x7fb914032420 secure :-1 s=READY pgs=24 cs=0 l=1 rev1=1 rx=0x7fb91c0096a0 tx=0x7fb91c006b40).stop 2023-11-17T05:22:55.633 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:55.631+0000 7fb92ffbe700 1 -- 172.21.15.66:0/4012507317 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fb928060280 msgr2=0x7fb9281066e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:22:55.633 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:55.631+0000 7fb92ffbe700 1 --2- 172.21.15.66:0/4012507317 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fb928060280 0x7fb9281066e0 secure :-1 s=READY pgs=118 cs=0 l=1 rev1=1 rx=0x7fb924013510 tx=0x7fb92400b040).stop 2023-11-17T05:22:55.633 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:55.631+0000 7fb92ffbe700 1 -- 172.21.15.66:0/4012507317 shutdown_connections 2023-11-17T05:22:55.633 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:55.631+0000 7fb92ffbe700 1 --2- 172.21.15.66:0/4012507317 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fb91402ff60 0x7fb914032420 unknown :-1 s=CLOSED pgs=24 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:22:55.633 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:55.631+0000 7fb92ffbe700 1 --2- 172.21.15.66:0/4012507317 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fb928060280 0x7fb9281066e0 unknown :-1 s=CLOSED pgs=118 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:22:55.633 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:55.631+0000 7fb92ffbe700 1 -- 172.21.15.66:0/4012507317 >> 172.21.15.66:0/4012507317 conn(0x7fb9280f0b70 msgr2=0x7fb9280f1660 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:22:55.633 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:55.631+0000 7fb92ffbe700 1 -- 172.21.15.66:0/4012507317 shutdown_connections 2023-11-17T05:22:55.633 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:55.632+0000 7fb92ffbe700 1 -- 172.21.15.66:0/4012507317 wait complete. 2023-11-17T05:22:55.634 INFO:teuthology.orchestra.run.smithi066.stderr:dumped monmap epoch 1 2023-11-17T05:22:56.740 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:22:56 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:22:55.172130+0000 mgr.smithi040.ogcprl (mgr.14164) 53 : cluster [DBG] pgmap v46: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:22:56.740 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:22:56 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:22:55.631550+0000 mon.smithi040 (mon.0) 134 : audit [DBG] from='client.? 172.21.15.66:0/4012507317' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-11-17T05:22:57.036 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2023-11-17T05:22:57.036 DEBUG:teuthology.orchestra.run.smithi066:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.4 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ed6470e2-8508-11ee-8dbe-212e2dc638e7 -- ceph mon dump -f json 2023-11-17T05:22:57.989 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:22:57 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:22:57.172502+0000 mgr.smithi040.ogcprl (mgr.14164) 54 : cluster [DBG] pgmap v47: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:22:58.297 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:58.294+0000 7fe9d4572700 1 -- 172.21.15.66:0/161316466 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fe9cc0f6510 msgr2=0x7fe9cc0f6930 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:22:58.297 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:58.294+0000 7fe9d4572700 1 --2- 172.21.15.66:0/161316466 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fe9cc0f6510 0x7fe9cc0f6930 secure :-1 s=READY pgs=119 cs=0 l=1 rev1=1 rx=0x7fe9c0009fb0 tx=0x7fe9c0009b30).stop 2023-11-17T05:22:58.297 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:58.294+0000 7fe9d4572700 1 -- 172.21.15.66:0/161316466 shutdown_connections 2023-11-17T05:22:58.297 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:58.294+0000 7fe9d4572700 1 --2- 172.21.15.66:0/161316466 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fe9cc0f6510 0x7fe9cc0f6930 unknown :-1 s=CLOSED pgs=119 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:22:58.297 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:58.294+0000 7fe9d4572700 1 -- 172.21.15.66:0/161316466 >> 172.21.15.66:0/161316466 conn(0x7fe9cc0f1c70 msgr2=0x7fe9cc0f4090 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:22:58.298 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:58.294+0000 7fe9d4572700 1 -- 172.21.15.66:0/161316466 shutdown_connections 2023-11-17T05:22:58.298 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:58.294+0000 7fe9d4572700 1 -- 172.21.15.66:0/161316466 wait complete. 2023-11-17T05:22:58.298 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:58.295+0000 7fe9d4572700 1 Processor -- start 2023-11-17T05:22:58.298 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:58.295+0000 7fe9d4572700 1 -- start start 2023-11-17T05:22:58.298 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:58.295+0000 7fe9d4572700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fe9cc0f6510 0x7fe9cc106730 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:22:58.298 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:58.295+0000 7fe9d4572700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fe9cc106c20 con 0x7fe9cc0f6510 2023-11-17T05:22:58.298 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:58.295+0000 7fe9d230e700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fe9cc0f6510 0x7fe9cc106730 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:22:58.298 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:58.295+0000 7fe9d230e700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fe9cc0f6510 0x7fe9cc106730 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.66:33294/0 (socket says 172.21.15.66:33294) 2023-11-17T05:22:58.298 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:58.295+0000 7fe9d230e700 1 -- 172.21.15.66:0/2740356517 learned_addr learned my addr 172.21.15.66:0/2740356517 (peer_addr_for_me v2:172.21.15.66:0/0) 2023-11-17T05:22:58.299 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:58.295+0000 7fe9d230e700 1 -- 172.21.15.66:0/2740356517 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fe9c0005040 con 0x7fe9cc0f6510 2023-11-17T05:22:58.299 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:58.296+0000 7fe9d230e700 1 --2- 172.21.15.66:0/2740356517 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fe9cc0f6510 0x7fe9cc106730 secure :-1 s=READY pgs=120 cs=0 l=1 rev1=1 rx=0x7fe9c0014000 tx=0x7fe9c000b040).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:22:58.299 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:58.296+0000 7fe9beffd700 1 -- 172.21.15.66:0/2740356517 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7fe9c00095f0 con 0x7fe9cc0f6510 2023-11-17T05:22:58.300 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:58.296+0000 7fe9d4572700 1 -- 172.21.15.66:0/2740356517 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fe9cc106e20 con 0x7fe9cc0f6510 2023-11-17T05:22:58.300 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:58.296+0000 7fe9beffd700 1 -- 172.21.15.66:0/2740356517 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(25 keys) v1 ==== 1074+0+0 (secure 0 0 0) 0x7fe9c0007e70 con 0x7fe9cc0f6510 2023-11-17T05:22:58.300 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:58.296+0000 7fe9beffd700 1 -- 172.21.15.66:0/2740356517 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7fe9c0014ea0 con 0x7fe9cc0f6510 2023-11-17T05:22:58.300 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:58.296+0000 7fe9d4572700 1 -- 172.21.15.66:0/2740356517 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fe9cc1072b0 con 0x7fe9cc0f6510 2023-11-17T05:22:58.300 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:58.297+0000 7fe9beffd700 1 -- 172.21.15.66:0/2740356517 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 17) v1 ==== 36269+0+0 (secure 0 0 0) 0x7fe9c001d4e0 con 0x7fe9cc0f6510 2023-11-17T05:22:58.300 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:58.297+0000 7fe9beffd700 1 --2- 172.21.15.66:0/2740356517 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fe9b802ff60 0x7fe9b8032420 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:22:58.300 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:58.297+0000 7fe9beffd700 1 -- 172.21.15.66:0/2740356517 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(5..5 src has 1..5) v4 ==== 1270+0+0 (secure 0 0 0) 0x7fe9c00408e0 con 0x7fe9cc0f6510 2023-11-17T05:22:58.301 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:58.297+0000 7fe9d4572700 1 -- 172.21.15.66:0/2740356517 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fe9cc045ad0 con 0x7fe9cc0f6510 2023-11-17T05:22:58.301 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:58.297+0000 7fe9d1b0d700 1 --2- 172.21.15.66:0/2740356517 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fe9b802ff60 0x7fe9b8032420 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:22:58.301 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:58.297+0000 7fe9d1b0d700 1 --2- 172.21.15.66:0/2740356517 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fe9b802ff60 0x7fe9b8032420 secure :-1 s=READY pgs=25 cs=0 l=1 rev1=1 rx=0x7fe9c80096a0 tx=0x7fe9c8006b40).ready entity=mgr.14164 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:22:58.303 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:58.302+0000 7fe9beffd700 1 -- 172.21.15.66:0/2740356517 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+152212 (secure 0 0 0) 0x7fe9c000f8f0 con 0x7fe9cc0f6510 2023-11-17T05:22:58.521 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:58.520+0000 7fe9d4572700 1 -- 172.21.15.66:0/2740356517 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "mon dump", "format": "json"} v 0) v1 -- 0x7fe9cc0fc050 con 0x7fe9cc0f6510 2023-11-17T05:22:58.522 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:58.520+0000 7fe9beffd700 1 -- 172.21.15.66:0/2740356517 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mon_command_ack([{"prefix": "mon dump", "format": "json"}]=0 dumped monmap epoch 1 v1) v1 ==== 95+0+690 (secure 0 0 0) 0x7fe9c0012330 con 0x7fe9cc0f6510 2023-11-17T05:22:58.522 INFO:teuthology.orchestra.run.smithi066.stdout: 2023-11-17T05:22:58.522 INFO:teuthology.orchestra.run.smithi066.stdout:{"epoch":1,"fsid":"ed6470e2-8508-11ee-8dbe-212e2dc638e7","modified":"2023-11-17T05:20:13.934038Z","created":"2023-11-17T05:20:13.934038Z","min_mon_release":16,"min_mon_release_name":"pacific","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging"],"optional":[]},"mons":[{"rank":0,"name":"smithi040","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:3300","nonce":0},{"type":"v1","addr":"172.21.15.40:6789","nonce":0}]},"addr":"172.21.15.40:6789/0","public_addr":"172.21.15.40:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2023-11-17T05:22:58.524 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:58.522+0000 7fe9d4572700 1 -- 172.21.15.66:0/2740356517 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fe9b802ff60 msgr2=0x7fe9b8032420 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:22:58.524 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:58.522+0000 7fe9d4572700 1 --2- 172.21.15.66:0/2740356517 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fe9b802ff60 0x7fe9b8032420 secure :-1 s=READY pgs=25 cs=0 l=1 rev1=1 rx=0x7fe9c80096a0 tx=0x7fe9c8006b40).stop 2023-11-17T05:22:58.524 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:58.522+0000 7fe9d4572700 1 -- 172.21.15.66:0/2740356517 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fe9cc0f6510 msgr2=0x7fe9cc106730 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:22:58.524 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:58.522+0000 7fe9d4572700 1 --2- 172.21.15.66:0/2740356517 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fe9cc0f6510 0x7fe9cc106730 secure :-1 s=READY pgs=120 cs=0 l=1 rev1=1 rx=0x7fe9c0014000 tx=0x7fe9c000b040).stop 2023-11-17T05:22:58.524 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:58.522+0000 7fe9d4572700 1 -- 172.21.15.66:0/2740356517 shutdown_connections 2023-11-17T05:22:58.524 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:58.522+0000 7fe9d4572700 1 --2- 172.21.15.66:0/2740356517 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fe9b802ff60 0x7fe9b8032420 unknown :-1 s=CLOSED pgs=25 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:22:58.524 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:58.522+0000 7fe9d4572700 1 --2- 172.21.15.66:0/2740356517 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fe9cc0f6510 0x7fe9cc106730 unknown :-1 s=CLOSED pgs=120 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:22:58.524 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:58.522+0000 7fe9d4572700 1 -- 172.21.15.66:0/2740356517 >> 172.21.15.66:0/2740356517 conn(0x7fe9cc0f1c70 msgr2=0x7fe9cc0f2760 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:22:58.525 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:58.522+0000 7fe9d4572700 1 -- 172.21.15.66:0/2740356517 shutdown_connections 2023-11-17T05:22:58.525 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:22:58.522+0000 7fe9d4572700 1 -- 172.21.15.66:0/2740356517 wait complete. 2023-11-17T05:22:58.525 INFO:teuthology.orchestra.run.smithi066.stderr:dumped monmap epoch 1 2023-11-17T05:22:58.989 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:22:58 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:22:58.521937+0000 mon.smithi040 (mon.0) 135 : audit [DBG] from='client.? 172.21.15.66:0/2740356517' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-11-17T05:22:59.912 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2023-11-17T05:22:59.912 DEBUG:teuthology.orchestra.run.smithi066:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.4 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ed6470e2-8508-11ee-8dbe-212e2dc638e7 -- ceph mon dump -f json 2023-11-17T05:23:00.739 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:23:00 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:22:59.172943+0000 mgr.smithi040.ogcprl (mgr.14164) 55 : cluster [DBG] pgmap v48: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:23:01.143 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:01.136+0000 7f98c9c3b700 1 -- 172.21.15.66:0/1472496644 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f98c4060220 msgr2=0x7f98c40f9150 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:23:01.143 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:01.136+0000 7f98c9c3b700 1 --2- 172.21.15.66:0/1472496644 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f98c4060220 0x7f98c40f9150 secure :-1 s=READY pgs=121 cs=0 l=1 rev1=1 rx=0x7f98bc009960 tx=0x7f98bc004d10).stop 2023-11-17T05:23:01.144 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:01.136+0000 7f98c9c3b700 1 -- 172.21.15.66:0/1472496644 shutdown_connections 2023-11-17T05:23:01.144 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:01.136+0000 7f98c9c3b700 1 --2- 172.21.15.66:0/1472496644 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f98c4060220 0x7f98c40f9150 unknown :-1 s=CLOSED pgs=121 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:23:01.144 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:01.136+0000 7f98c9c3b700 1 -- 172.21.15.66:0/1472496644 >> 172.21.15.66:0/1472496644 conn(0x7f98c40f0b70 msgr2=0x7f98c40f2f90 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:23:01.144 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:01.136+0000 7f98c9c3b700 1 -- 172.21.15.66:0/1472496644 shutdown_connections 2023-11-17T05:23:01.144 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:01.136+0000 7f98c9c3b700 1 -- 172.21.15.66:0/1472496644 wait complete. 2023-11-17T05:23:01.144 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:01.137+0000 7f98c9c3b700 1 Processor -- start 2023-11-17T05:23:01.145 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:01.137+0000 7f98c9c3b700 1 -- start start 2023-11-17T05:23:01.145 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:01.137+0000 7f98c9c3b700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f98c4060220 0x7f98c41088f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:23:01.145 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:01.137+0000 7f98c9c3b700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f98c4108de0 con 0x7f98c4060220 2023-11-17T05:23:01.145 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:01.138+0000 7f98c37fe700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f98c4060220 0x7f98c41088f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:23:01.145 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:01.138+0000 7f98c37fe700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f98c4060220 0x7f98c41088f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.66:33318/0 (socket says 172.21.15.66:33318) 2023-11-17T05:23:01.146 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:01.138+0000 7f98c37fe700 1 -- 172.21.15.66:0/3933090295 learned_addr learned my addr 172.21.15.66:0/3933090295 (peer_addr_for_me v2:172.21.15.66:0/0) 2023-11-17T05:23:01.146 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:01.138+0000 7f98c37fe700 1 -- 172.21.15.66:0/3933090295 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f98bc009640 con 0x7f98c4060220 2023-11-17T05:23:01.162 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:01.138+0000 7f98c37fe700 1 --2- 172.21.15.66:0/3933090295 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f98c4060220 0x7f98c41088f0 secure :-1 s=READY pgs=122 cs=0 l=1 rev1=1 rx=0x7f98bc007bd0 tx=0x7f98bc004d10).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:23:01.162 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:01.138+0000 7f98abfff700 1 -- 172.21.15.66:0/3933090295 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f98bc018070 con 0x7f98c4060220 2023-11-17T05:23:01.162 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:01.138+0000 7f98c9c3b700 1 -- 172.21.15.66:0/3933090295 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f98c4108fe0 con 0x7f98c4060220 2023-11-17T05:23:01.162 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:01.138+0000 7f98abfff700 1 -- 172.21.15.66:0/3933090295 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(25 keys) v1 ==== 1074+0+0 (secure 0 0 0) 0x7f98bc00b870 con 0x7f98c4060220 2023-11-17T05:23:01.169 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:01.138+0000 7f98abfff700 1 -- 172.21.15.66:0/3933090295 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f98bc01d420 con 0x7f98c4060220 2023-11-17T05:23:01.169 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:01.138+0000 7f98c9c3b700 1 -- 172.21.15.66:0/3933090295 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f98c4109470 con 0x7f98c4060220 2023-11-17T05:23:01.169 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:01.139+0000 7f98abfff700 1 -- 172.21.15.66:0/3933090295 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 17) v1 ==== 36269+0+0 (secure 0 0 0) 0x7f98bc024060 con 0x7f98c4060220 2023-11-17T05:23:01.169 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:01.139+0000 7f98c9c3b700 1 -- 172.21.15.66:0/3933090295 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f98c4045ad0 con 0x7f98c4060220 2023-11-17T05:23:01.170 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:01.139+0000 7f98abfff700 1 --2- 172.21.15.66:0/3933090295 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f98ac02ff60 0x7f98ac032420 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:23:01.170 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:01.139+0000 7f98abfff700 1 -- 172.21.15.66:0/3933090295 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(5..5 src has 1..5) v4 ==== 1270+0+0 (secure 0 0 0) 0x7f98bc041140 con 0x7f98c4060220 2023-11-17T05:23:01.170 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:01.140+0000 7f98c2ffd700 1 --2- 172.21.15.66:0/3933090295 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f98ac02ff60 0x7f98ac032420 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:23:01.170 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:01.140+0000 7f98c2ffd700 1 --2- 172.21.15.66:0/3933090295 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f98ac02ff60 0x7f98ac032420 secure :-1 s=READY pgs=26 cs=0 l=1 rev1=1 rx=0x7f98b80096a0 tx=0x7f98b8006b40).ready entity=mgr.14164 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:23:01.170 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:01.145+0000 7f98abfff700 1 -- 172.21.15.66:0/3933090295 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+152212 (secure 0 0 0) 0x7f98bc025520 con 0x7f98c4060220 2023-11-17T05:23:01.365 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:01.363+0000 7f98c9c3b700 1 -- 172.21.15.66:0/3933090295 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "mon dump", "format": "json"} v 0) v1 -- 0x7f98c40fbf50 con 0x7f98c4060220 2023-11-17T05:23:01.365 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:01.364+0000 7f98abfff700 1 -- 172.21.15.66:0/3933090295 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mon_command_ack([{"prefix": "mon dump", "format": "json"}]=0 dumped monmap epoch 1 v1) v1 ==== 95+0+690 (secure 0 0 0) 0x7f98bc025340 con 0x7f98c4060220 2023-11-17T05:23:01.366 INFO:teuthology.orchestra.run.smithi066.stdout: 2023-11-17T05:23:01.366 INFO:teuthology.orchestra.run.smithi066.stdout:{"epoch":1,"fsid":"ed6470e2-8508-11ee-8dbe-212e2dc638e7","modified":"2023-11-17T05:20:13.934038Z","created":"2023-11-17T05:20:13.934038Z","min_mon_release":16,"min_mon_release_name":"pacific","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging"],"optional":[]},"mons":[{"rank":0,"name":"smithi040","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:3300","nonce":0},{"type":"v1","addr":"172.21.15.40:6789","nonce":0}]},"addr":"172.21.15.40:6789/0","public_addr":"172.21.15.40:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2023-11-17T05:23:01.367 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:01.366+0000 7f98c9c3b700 1 -- 172.21.15.66:0/3933090295 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f98ac02ff60 msgr2=0x7f98ac032420 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:23:01.368 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:01.366+0000 7f98c9c3b700 1 --2- 172.21.15.66:0/3933090295 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f98ac02ff60 0x7f98ac032420 secure :-1 s=READY pgs=26 cs=0 l=1 rev1=1 rx=0x7f98b80096a0 tx=0x7f98b8006b40).stop 2023-11-17T05:23:01.368 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:01.366+0000 7f98c9c3b700 1 -- 172.21.15.66:0/3933090295 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f98c4060220 msgr2=0x7f98c41088f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:23:01.368 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:01.366+0000 7f98c9c3b700 1 --2- 172.21.15.66:0/3933090295 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f98c4060220 0x7f98c41088f0 secure :-1 s=READY pgs=122 cs=0 l=1 rev1=1 rx=0x7f98bc007bd0 tx=0x7f98bc004d10).stop 2023-11-17T05:23:01.368 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:01.366+0000 7f98c9c3b700 1 -- 172.21.15.66:0/3933090295 shutdown_connections 2023-11-17T05:23:01.368 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:01.366+0000 7f98c9c3b700 1 --2- 172.21.15.66:0/3933090295 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f98ac02ff60 0x7f98ac032420 unknown :-1 s=CLOSED pgs=26 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:23:01.368 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:01.366+0000 7f98c9c3b700 1 --2- 172.21.15.66:0/3933090295 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f98c4060220 0x7f98c41088f0 unknown :-1 s=CLOSED pgs=122 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:23:01.368 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:01.366+0000 7f98c9c3b700 1 -- 172.21.15.66:0/3933090295 >> 172.21.15.66:0/3933090295 conn(0x7f98c40f0b70 msgr2=0x7f98c40f1620 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:23:01.368 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:01.366+0000 7f98c9c3b700 1 -- 172.21.15.66:0/3933090295 shutdown_connections 2023-11-17T05:23:01.368 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:01.366+0000 7f98c9c3b700 1 -- 172.21.15.66:0/3933090295 wait complete. 2023-11-17T05:23:01.369 INFO:teuthology.orchestra.run.smithi066.stderr:dumped monmap epoch 1 2023-11-17T05:23:01.739 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:23:01 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:23:01.365818+0000 mon.smithi040 (mon.0) 136 : audit [DBG] from='client.? 172.21.15.66:0/3933090295' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-11-17T05:23:02.739 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:23:02 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:23:01.173220+0000 mgr.smithi040.ogcprl (mgr.14164) 56 : cluster [DBG] pgmap v49: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:23:02.785 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2023-11-17T05:23:02.785 DEBUG:teuthology.orchestra.run.smithi066:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.4 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ed6470e2-8508-11ee-8dbe-212e2dc638e7 -- ceph mon dump -f json 2023-11-17T05:23:03.932 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:03.930+0000 7f93e955a700 1 -- 172.21.15.66:0/688184694 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f93e40f4b80 msgr2=0x7f93e40f4fa0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:23:03.932 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:03.930+0000 7f93e955a700 1 --2- 172.21.15.66:0/688184694 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f93e40f4b80 0x7f93e40f4fa0 secure :-1 s=READY pgs=123 cs=0 l=1 rev1=1 rx=0x7f93d4004770 tx=0x7f93d400ac40).stop 2023-11-17T05:23:03.932 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:03.930+0000 7f93e955a700 1 -- 172.21.15.66:0/688184694 shutdown_connections 2023-11-17T05:23:03.932 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:03.930+0000 7f93e955a700 1 --2- 172.21.15.66:0/688184694 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f93e40f4b80 0x7f93e40f4fa0 unknown :-1 s=CLOSED pgs=123 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:23:03.932 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:03.930+0000 7f93e955a700 1 -- 172.21.15.66:0/688184694 >> 172.21.15.66:0/688184694 conn(0x7f93e40f02e0 msgr2=0x7f93e40f2700 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:23:03.932 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:03.930+0000 7f93e955a700 1 -- 172.21.15.66:0/688184694 shutdown_connections 2023-11-17T05:23:03.932 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:03.930+0000 7f93e955a700 1 -- 172.21.15.66:0/688184694 wait complete. 2023-11-17T05:23:03.934 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:03.931+0000 7f93e955a700 1 Processor -- start 2023-11-17T05:23:03.934 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:03.931+0000 7f93e955a700 1 -- start start 2023-11-17T05:23:03.934 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:03.931+0000 7f93e955a700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f93e40f4b80 0x7f93e4106720 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:23:03.934 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:03.931+0000 7f93e955a700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f93e4106c10 con 0x7f93e40f4b80 2023-11-17T05:23:03.934 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:03.932+0000 7f93e2ffd700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f93e40f4b80 0x7f93e4106720 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:23:03.934 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:03.932+0000 7f93e2ffd700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f93e40f4b80 0x7f93e4106720 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.66:56618/0 (socket says 172.21.15.66:56618) 2023-11-17T05:23:03.934 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:03.932+0000 7f93e2ffd700 1 -- 172.21.15.66:0/4092950123 learned_addr learned my addr 172.21.15.66:0/4092950123 (peer_addr_for_me v2:172.21.15.66:0/0) 2023-11-17T05:23:03.934 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:03.932+0000 7f93e2ffd700 1 -- 172.21.15.66:0/4092950123 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f93d4005040 con 0x7f93e40f4b80 2023-11-17T05:23:03.934 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:03.932+0000 7f93e2ffd700 1 --2- 172.21.15.66:0/4092950123 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f93e40f4b80 0x7f93e4106720 secure :-1 s=READY pgs=124 cs=0 l=1 rev1=1 rx=0x7f93d4004740 tx=0x7f93d400ac40).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:23:03.934 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:03.932+0000 7f93cb7fe700 1 -- 172.21.15.66:0/4092950123 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f93d4019070 con 0x7f93e40f4b80 2023-11-17T05:23:03.935 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:03.932+0000 7f93e955a700 1 -- 172.21.15.66:0/4092950123 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f93e4106e10 con 0x7f93e40f4b80 2023-11-17T05:23:03.935 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:03.932+0000 7f93cb7fe700 1 -- 172.21.15.66:0/4092950123 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(25 keys) v1 ==== 1074+0+0 (secure 0 0 0) 0x7f93d4007e70 con 0x7f93e40f4b80 2023-11-17T05:23:03.935 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:03.932+0000 7f93cb7fe700 1 -- 172.21.15.66:0/4092950123 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f93d40082d0 con 0x7f93e40f4b80 2023-11-17T05:23:03.935 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:03.932+0000 7f93e955a700 1 -- 172.21.15.66:0/4092950123 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f93e41072a0 con 0x7f93e40f4b80 2023-11-17T05:23:03.936 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:03.933+0000 7f93cb7fe700 1 -- 172.21.15.66:0/4092950123 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 17) v1 ==== 36269+0+0 (secure 0 0 0) 0x7f93d4008520 con 0x7f93e40f4b80 2023-11-17T05:23:03.936 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:03.933+0000 7f93e955a700 1 -- 172.21.15.66:0/4092950123 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f93e4045ad0 con 0x7f93e40f4b80 2023-11-17T05:23:03.936 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:03.933+0000 7f93cb7fe700 1 --2- 172.21.15.66:0/4092950123 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f93cc02ff60 0x7f93cc032420 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:23:03.936 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:03.933+0000 7f93cb7fe700 1 -- 172.21.15.66:0/4092950123 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(5..5 src has 1..5) v4 ==== 1270+0+0 (secure 0 0 0) 0x7f93d4040cc0 con 0x7f93e40f4b80 2023-11-17T05:23:03.936 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:03.934+0000 7f93e27fc700 1 --2- 172.21.15.66:0/4092950123 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f93cc02ff60 0x7f93cc032420 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:23:03.936 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:03.934+0000 7f93e27fc700 1 --2- 172.21.15.66:0/4092950123 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f93cc02ff60 0x7f93cc032420 secure :-1 s=READY pgs=27 cs=0 l=1 rev1=1 rx=0x7f93d80096a0 tx=0x7f93d8006b40).ready entity=mgr.14164 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:23:03.939 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:03.938+0000 7f93cb7fe700 1 -- 172.21.15.66:0/4092950123 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+152212 (secure 0 0 0) 0x7f93d4009e80 con 0x7f93e40f4b80 2023-11-17T05:23:04.160 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:04.159+0000 7f93e955a700 1 -- 172.21.15.66:0/4092950123 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "mon dump", "format": "json"} v 0) v1 -- 0x7f93e40f9d40 con 0x7f93e40f4b80 2023-11-17T05:23:04.161 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:04.160+0000 7f93cb7fe700 1 -- 172.21.15.66:0/4092950123 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mon_command_ack([{"prefix": "mon dump", "format": "json"}]=0 dumped monmap epoch 1 v1) v1 ==== 95+0+690 (secure 0 0 0) 0x7f93d4023250 con 0x7f93e40f4b80 2023-11-17T05:23:04.161 INFO:teuthology.orchestra.run.smithi066.stdout: 2023-11-17T05:23:04.161 INFO:teuthology.orchestra.run.smithi066.stdout:{"epoch":1,"fsid":"ed6470e2-8508-11ee-8dbe-212e2dc638e7","modified":"2023-11-17T05:20:13.934038Z","created":"2023-11-17T05:20:13.934038Z","min_mon_release":16,"min_mon_release_name":"pacific","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging"],"optional":[]},"mons":[{"rank":0,"name":"smithi040","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:3300","nonce":0},{"type":"v1","addr":"172.21.15.40:6789","nonce":0}]},"addr":"172.21.15.40:6789/0","public_addr":"172.21.15.40:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2023-11-17T05:23:04.163 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:04.161+0000 7f93e955a700 1 -- 172.21.15.66:0/4092950123 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f93cc02ff60 msgr2=0x7f93cc032420 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:23:04.163 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:04.161+0000 7f93e955a700 1 --2- 172.21.15.66:0/4092950123 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f93cc02ff60 0x7f93cc032420 secure :-1 s=READY pgs=27 cs=0 l=1 rev1=1 rx=0x7f93d80096a0 tx=0x7f93d8006b40).stop 2023-11-17T05:23:04.163 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:04.161+0000 7f93e955a700 1 -- 172.21.15.66:0/4092950123 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f93e40f4b80 msgr2=0x7f93e4106720 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:23:04.163 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:04.161+0000 7f93e955a700 1 --2- 172.21.15.66:0/4092950123 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f93e40f4b80 0x7f93e4106720 secure :-1 s=READY pgs=124 cs=0 l=1 rev1=1 rx=0x7f93d4004740 tx=0x7f93d400ac40).stop 2023-11-17T05:23:04.163 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:04.161+0000 7f93e955a700 1 -- 172.21.15.66:0/4092950123 shutdown_connections 2023-11-17T05:23:04.163 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:04.161+0000 7f93e955a700 1 --2- 172.21.15.66:0/4092950123 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f93cc02ff60 0x7f93cc032420 unknown :-1 s=CLOSED pgs=27 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:23:04.163 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:04.161+0000 7f93e955a700 1 --2- 172.21.15.66:0/4092950123 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f93e40f4b80 0x7f93e4106720 unknown :-1 s=CLOSED pgs=124 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:23:04.163 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:04.161+0000 7f93e955a700 1 -- 172.21.15.66:0/4092950123 >> 172.21.15.66:0/4092950123 conn(0x7f93e40f02e0 msgr2=0x7f93e40f0dd0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:23:04.163 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:04.161+0000 7f93e955a700 1 -- 172.21.15.66:0/4092950123 shutdown_connections 2023-11-17T05:23:04.163 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:04.161+0000 7f93e955a700 1 -- 172.21.15.66:0/4092950123 wait complete. 2023-11-17T05:23:04.164 INFO:teuthology.orchestra.run.smithi066.stderr:dumped monmap epoch 1 2023-11-17T05:23:04.739 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:23:04 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:23:03.173615+0000 mgr.smithi040.ogcprl (mgr.14164) 57 : cluster [DBG] pgmap v50: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:23:04.739 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:23:04 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:23:04.161219+0000 mon.smithi040 (mon.0) 137 : audit [DBG] from='client.? 172.21.15.66:0/4092950123' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-11-17T05:23:05.578 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2023-11-17T05:23:05.578 DEBUG:teuthology.orchestra.run.smithi066:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.4 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ed6470e2-8508-11ee-8dbe-212e2dc638e7 -- ceph mon dump -f json 2023-11-17T05:23:06.739 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:23:06 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:23:05.174109+0000 mgr.smithi040.ogcprl (mgr.14164) 58 : cluster [DBG] pgmap v51: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:23:13.915 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:23:07 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:23:07.043513+0000 mon.smithi040 (mon.0) 138 : audit [DBG] from='client.? 172.21.15.66:0/3552950270' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-11-17T05:23:13.916 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:23:08 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:23:07.174448+0000 mgr.smithi040.ogcprl (mgr.14164) 59 : cluster [DBG] pgmap v52: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:23:13.916 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:23:10 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:23:09.174740+0000 mgr.smithi040.ogcprl (mgr.14164) 60 : cluster [DBG] pgmap v53: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:23:13.916 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:23:11 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:23:11.308596+0000 mon.smithi040 (mon.0) 139 : audit [INF] from='mgr.14164 172.21.15.40:0/2203032451' entity='mgr.smithi040.ogcprl' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi040.ogcprl/mirror_snapshot_schedule"}]: dispatch 2023-11-17T05:23:13.916 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:23:11 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:23:11.311070+0000 mon.smithi040 (mon.0) 140 : audit [INF] from='mgr.14164 172.21.15.40:0/2203032451' entity='mgr.smithi040.ogcprl' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi040.ogcprl/trash_purge_schedule"}]: dispatch 2023-11-17T05:23:13.916 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:23:12 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:23:11.174973+0000 mgr.smithi040.ogcprl (mgr.14164) 61 : cluster [DBG] pgmap v54: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:23:13.917 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:06.815+0000 7fbdad35e700 1 -- 172.21.15.66:0/2955783471 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fbda8060280 msgr2=0x7fbda80f90f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:23:13.917 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:06.815+0000 7fbdad35e700 1 --2- 172.21.15.66:0/2955783471 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fbda8060280 0x7fbda80f90f0 secure :-1 s=READY pgs=125 cs=0 l=1 rev1=1 rx=0x7fbd98009fb0 tx=0x7fbd98009b30).stop 2023-11-17T05:23:13.918 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:06.815+0000 7fbdad35e700 1 -- 172.21.15.66:0/2955783471 shutdown_connections 2023-11-17T05:23:13.918 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:06.815+0000 7fbdad35e700 1 --2- 172.21.15.66:0/2955783471 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fbda8060280 0x7fbda80f90f0 unknown :-1 s=CLOSED pgs=125 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:23:13.918 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:06.815+0000 7fbdad35e700 1 -- 172.21.15.66:0/2955783471 >> 172.21.15.66:0/2955783471 conn(0x7fbda80f0b70 msgr2=0x7fbda80f2f90 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:23:13.918 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:06.815+0000 7fbdad35e700 1 -- 172.21.15.66:0/2955783471 shutdown_connections 2023-11-17T05:23:13.918 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:06.815+0000 7fbdad35e700 1 -- 172.21.15.66:0/2955783471 wait complete. 2023-11-17T05:23:13.918 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:06.816+0000 7fbdad35e700 1 Processor -- start 2023-11-17T05:23:13.918 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:06.816+0000 7fbdad35e700 1 -- start start 2023-11-17T05:23:13.918 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:06.816+0000 7fbdad35e700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fbda8060280 0x7fbda80f6b70 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:23:13.918 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:06.816+0000 7fbdad35e700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fbda80f4ff0 con 0x7fbda8060280 2023-11-17T05:23:13.918 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:06.817+0000 7fbda6ffd700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fbda8060280 0x7fbda80f6b70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:23:13.918 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:06.817+0000 7fbda6ffd700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fbda8060280 0x7fbda80f6b70 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.66:56624/0 (socket says 172.21.15.66:56624) 2023-11-17T05:23:13.918 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:06.817+0000 7fbda6ffd700 1 -- 172.21.15.66:0/3552950270 learned_addr learned my addr 172.21.15.66:0/3552950270 (peer_addr_for_me v2:172.21.15.66:0/0) 2023-11-17T05:23:13.918 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:06.817+0000 7fbda6ffd700 1 -- 172.21.15.66:0/3552950270 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fbd98005040 con 0x7fbda8060280 2023-11-17T05:23:13.919 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:06.817+0000 7fbda6ffd700 1 --2- 172.21.15.66:0/3552950270 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fbda8060280 0x7fbda80f6b70 secure :-1 s=READY pgs=126 cs=0 l=1 rev1=1 rx=0x7fbd98013510 tx=0x7fbd9800b040).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:23:13.919 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:06.817+0000 7fbd9f7fe700 1 -- 172.21.15.66:0/3552950270 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7fbd98009320 con 0x7fbda8060280 2023-11-17T05:23:13.919 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:06.817+0000 7fbdad35e700 1 -- 172.21.15.66:0/3552950270 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fbda80f5190 con 0x7fbda8060280 2023-11-17T05:23:13.945 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:06.817+0000 7fbd9f7fe700 1 -- 172.21.15.66:0/3552950270 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(25 keys) v1 ==== 1074+0+0 (secure 0 0 0) 0x7fbd98013f00 con 0x7fbda8060280 2023-11-17T05:23:13.946 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:06.817+0000 7fbd9f7fe700 1 -- 172.21.15.66:0/3552950270 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7fbd98014dc0 con 0x7fbda8060280 2023-11-17T05:23:13.946 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:06.817+0000 7fbdad35e700 1 -- 172.21.15.66:0/3552950270 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fbda80f5620 con 0x7fbda8060280 2023-11-17T05:23:13.946 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:06.818+0000 7fbd9f7fe700 1 -- 172.21.15.66:0/3552950270 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 17) v1 ==== 36269+0+0 (secure 0 0 0) 0x7fbd9801d3d0 con 0x7fbda8060280 2023-11-17T05:23:13.947 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:06.818+0000 7fbdad35e700 1 -- 172.21.15.66:0/3552950270 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fbda8059100 con 0x7fbda8060280 2023-11-17T05:23:13.947 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:06.818+0000 7fbd9f7fe700 1 --2- 172.21.15.66:0/3552950270 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fbd9402ff60 0x7fbd94032420 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:23:13.947 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:06.818+0000 7fbd9f7fe700 1 -- 172.21.15.66:0/3552950270 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(5..5 src has 1..5) v4 ==== 1270+0+0 (secure 0 0 0) 0x7fbd980413e0 con 0x7fbda8060280 2023-11-17T05:23:13.949 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:06.818+0000 7fbda67fc700 1 --2- 172.21.15.66:0/3552950270 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fbd9402ff60 0x7fbd94032420 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:23:13.949 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:06.819+0000 7fbda67fc700 1 --2- 172.21.15.66:0/3552950270 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fbd9402ff60 0x7fbd94032420 secure :-1 s=READY pgs=28 cs=0 l=1 rev1=1 rx=0x7fbd900096a0 tx=0x7fbd90006b40).ready entity=mgr.14164 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:23:13.949 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:06.823+0000 7fbd9f7fe700 1 -- 172.21.15.66:0/3552950270 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+152212 (secure 0 0 0) 0x7fbd9801cb10 con 0x7fbda8060280 2023-11-17T05:23:13.949 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:07.041+0000 7fbdad35e700 1 -- 172.21.15.66:0/3552950270 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "mon dump", "format": "json"} v 0) v1 -- 0x7fbda8045ad0 con 0x7fbda8060280 2023-11-17T05:23:13.950 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:07.043+0000 7fbd9f7fe700 1 -- 172.21.15.66:0/3552950270 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mon_command_ack([{"prefix": "mon dump", "format": "json"}]=0 dumped monmap epoch 1 v1) v1 ==== 95+0+690 (secure 0 0 0) 0x7fbd9801cb10 con 0x7fbda8060280 2023-11-17T05:23:13.954 INFO:teuthology.orchestra.run.smithi066.stdout: 2023-11-17T05:23:13.954 INFO:teuthology.orchestra.run.smithi066.stdout:{"epoch":1,"fsid":"ed6470e2-8508-11ee-8dbe-212e2dc638e7","modified":"2023-11-17T05:20:13.934038Z","created":"2023-11-17T05:20:13.934038Z","min_mon_release":16,"min_mon_release_name":"pacific","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging"],"optional":[]},"mons":[{"rank":0,"name":"smithi040","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:3300","nonce":0},{"type":"v1","addr":"172.21.15.40:6789","nonce":0}]},"addr":"172.21.15.40:6789/0","public_addr":"172.21.15.40:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2023-11-17T05:23:13.961 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:07.044+0000 7fbdad35e700 1 -- 172.21.15.66:0/3552950270 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fbd9402ff60 msgr2=0x7fbd94032420 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:23:13.962 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:07.044+0000 7fbdad35e700 1 --2- 172.21.15.66:0/3552950270 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fbd9402ff60 0x7fbd94032420 secure :-1 s=READY pgs=28 cs=0 l=1 rev1=1 rx=0x7fbd900096a0 tx=0x7fbd90006b40).stop 2023-11-17T05:23:13.964 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:07.044+0000 7fbdad35e700 1 -- 172.21.15.66:0/3552950270 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fbda8060280 msgr2=0x7fbda80f6b70 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:23:13.965 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:07.044+0000 7fbdad35e700 1 --2- 172.21.15.66:0/3552950270 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fbda8060280 0x7fbda80f6b70 secure :-1 s=READY pgs=126 cs=0 l=1 rev1=1 rx=0x7fbd98013510 tx=0x7fbd9800b040).stop 2023-11-17T05:23:13.966 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:07.044+0000 7fbdad35e700 1 -- 172.21.15.66:0/3552950270 shutdown_connections 2023-11-17T05:23:13.967 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:07.044+0000 7fbdad35e700 1 --2- 172.21.15.66:0/3552950270 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fbd9402ff60 0x7fbd94032420 unknown :-1 s=CLOSED pgs=28 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:23:13.968 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:07.044+0000 7fbdad35e700 1 --2- 172.21.15.66:0/3552950270 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fbda8060280 0x7fbda80f6b70 unknown :-1 s=CLOSED pgs=126 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:23:13.969 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:07.044+0000 7fbdad35e700 1 -- 172.21.15.66:0/3552950270 >> 172.21.15.66:0/3552950270 conn(0x7fbda80f0b70 msgr2=0x7fbda80f1660 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:23:13.971 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:07.044+0000 7fbdad35e700 1 -- 172.21.15.66:0/3552950270 shutdown_connections 2023-11-17T05:23:13.971 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:07.044+0000 7fbdad35e700 1 -- 172.21.15.66:0/3552950270 wait complete. 2023-11-17T05:23:13.972 INFO:teuthology.orchestra.run.smithi066.stderr:dumped monmap epoch 1 2023-11-17T05:23:14.742 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:23:14 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:23:13.175247+0000 mgr.smithi040.ogcprl (mgr.14164) 62 : cluster [DBG] pgmap v55: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:23:14.974 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2023-11-17T05:23:14.975 DEBUG:teuthology.orchestra.run.smithi066:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.4 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ed6470e2-8508-11ee-8dbe-212e2dc638e7 -- ceph mon dump -f json 2023-11-17T05:23:16.162 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:16.159+0000 7f0bfb59e700 1 -- 172.21.15.66:0/4122508362 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f0bfc0f4920 msgr2=0x7f0bfc0f4d40 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:23:16.162 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:16.159+0000 7f0bfb59e700 1 --2- 172.21.15.66:0/4122508362 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f0bfc0f4920 0x7f0bfc0f4d40 secure :-1 s=READY pgs=127 cs=0 l=1 rev1=1 rx=0x7f0bec009fb0 tx=0x7f0bec009b30).stop 2023-11-17T05:23:16.162 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:16.159+0000 7f0bfb59e700 1 -- 172.21.15.66:0/4122508362 shutdown_connections 2023-11-17T05:23:16.162 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:16.159+0000 7f0bfb59e700 1 --2- 172.21.15.66:0/4122508362 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f0bfc0f4920 0x7f0bfc0f4d40 unknown :-1 s=CLOSED pgs=127 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:23:16.162 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:16.159+0000 7f0bfb59e700 1 -- 172.21.15.66:0/4122508362 >> 172.21.15.66:0/4122508362 conn(0x7f0bfc0f0120 msgr2=0x7f0bfc0f2540 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:23:16.163 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:16.159+0000 7f0bfb59e700 1 -- 172.21.15.66:0/4122508362 shutdown_connections 2023-11-17T05:23:16.163 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:16.159+0000 7f0bfb59e700 1 -- 172.21.15.66:0/4122508362 wait complete. 2023-11-17T05:23:16.163 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:16.159+0000 7f0bfb59e700 1 Processor -- start 2023-11-17T05:23:16.163 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:16.159+0000 7f0bfb59e700 1 -- start start 2023-11-17T05:23:16.163 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:16.160+0000 7f0bfb59e700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f0bfc0f4920 0x7f0bfc1091d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:23:16.163 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:16.160+0000 7f0bfb59e700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f0bfc0f16e0 con 0x7f0bfc0f4920 2023-11-17T05:23:16.163 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:16.160+0000 7f0bfa59c700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f0bfc0f4920 0x7f0bfc1091d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:23:16.163 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:16.160+0000 7f0bfa59c700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f0bfc0f4920 0x7f0bfc1091d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.66:39160/0 (socket says 172.21.15.66:39160) 2023-11-17T05:23:16.163 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:16.160+0000 7f0bfa59c700 1 -- 172.21.15.66:0/514962923 learned_addr learned my addr 172.21.15.66:0/514962923 (peer_addr_for_me v2:172.21.15.66:0/0) 2023-11-17T05:23:16.163 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:16.160+0000 7f0bfa59c700 1 -- 172.21.15.66:0/514962923 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f0bec005040 con 0x7f0bfc0f4920 2023-11-17T05:23:16.163 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:16.160+0000 7f0bfa59c700 1 --2- 172.21.15.66:0/514962923 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f0bfc0f4920 0x7f0bfc1091d0 secure :-1 s=READY pgs=128 cs=0 l=1 rev1=1 rx=0x7f0bec013510 tx=0x7f0bec00b040).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:23:16.164 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:16.160+0000 7f0beaffd700 1 -- 172.21.15.66:0/514962923 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f0bec009320 con 0x7f0bfc0f4920 2023-11-17T05:23:16.164 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:16.160+0000 7f0bfb59e700 1 -- 172.21.15.66:0/514962923 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f0bfc0f18e0 con 0x7f0bfc0f4920 2023-11-17T05:23:16.164 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:16.161+0000 7f0beaffd700 1 -- 172.21.15.66:0/514962923 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(25 keys) v1 ==== 1074+0+0 (secure 0 0 0) 0x7f0bec013f00 con 0x7f0bfc0f4920 2023-11-17T05:23:16.164 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:16.161+0000 7f0beaffd700 1 -- 172.21.15.66:0/514962923 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f0bec014dc0 con 0x7f0bfc0f4920 2023-11-17T05:23:16.164 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:16.161+0000 7f0bfb59e700 1 -- 172.21.15.66:0/514962923 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f0bfc109880 con 0x7f0bfc0f4920 2023-11-17T05:23:16.165 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:16.161+0000 7f0beaffd700 1 -- 172.21.15.66:0/514962923 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 17) v1 ==== 36269+0+0 (secure 0 0 0) 0x7f0bec0095f0 con 0x7f0bfc0f4920 2023-11-17T05:23:16.165 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:16.161+0000 7f0bfb59e700 1 -- 172.21.15.66:0/514962923 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f0bfc059100 con 0x7f0bfc0f4920 2023-11-17T05:23:16.165 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:16.161+0000 7f0beaffd700 1 --2- 172.21.15.66:0/514962923 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f0be402ff60 0x7f0be4032420 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:23:16.165 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:16.162+0000 7f0beaffd700 1 -- 172.21.15.66:0/514962923 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(5..5 src has 1..5) v4 ==== 1270+0+0 (secure 0 0 0) 0x7f0bec0410c0 con 0x7f0bfc0f4920 2023-11-17T05:23:16.165 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:16.162+0000 7f0bf9d9b700 1 --2- 172.21.15.66:0/514962923 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f0be402ff60 0x7f0be4032420 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:23:16.165 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:16.162+0000 7f0bf9d9b700 1 --2- 172.21.15.66:0/514962923 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f0be402ff60 0x7f0be4032420 secure :-1 s=READY pgs=29 cs=0 l=1 rev1=1 rx=0x7f0bf00096a0 tx=0x7f0bf0006b40).ready entity=mgr.14164 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:23:16.170 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:16.166+0000 7f0beaffd700 1 -- 172.21.15.66:0/514962923 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+152212 (secure 0 0 0) 0x7f0bec014420 con 0x7f0bfc0f4920 2023-11-17T05:23:16.386 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:16.384+0000 7f0bfb59e700 1 -- 172.21.15.66:0/514962923 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "mon dump", "format": "json"} v 0) v1 -- 0x7f0bfc045ad0 con 0x7f0bfc0f4920 2023-11-17T05:23:16.386 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:16.386+0000 7f0beaffd700 1 -- 172.21.15.66:0/514962923 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mon_command_ack([{"prefix": "mon dump", "format": "json"}]=0 dumped monmap epoch 1 v1) v1 ==== 95+0+690 (secure 0 0 0) 0x7f0bec010230 con 0x7f0bfc0f4920 2023-11-17T05:23:16.386 INFO:teuthology.orchestra.run.smithi066.stdout: 2023-11-17T05:23:16.386 INFO:teuthology.orchestra.run.smithi066.stdout:{"epoch":1,"fsid":"ed6470e2-8508-11ee-8dbe-212e2dc638e7","modified":"2023-11-17T05:20:13.934038Z","created":"2023-11-17T05:20:13.934038Z","min_mon_release":16,"min_mon_release_name":"pacific","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging"],"optional":[]},"mons":[{"rank":0,"name":"smithi040","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:3300","nonce":0},{"type":"v1","addr":"172.21.15.40:6789","nonce":0}]},"addr":"172.21.15.40:6789/0","public_addr":"172.21.15.40:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2023-11-17T05:23:16.388 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:16.386+0000 7f0bfb59e700 1 -- 172.21.15.66:0/514962923 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f0be402ff60 msgr2=0x7f0be4032420 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:23:16.388 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:16.386+0000 7f0bfb59e700 1 --2- 172.21.15.66:0/514962923 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f0be402ff60 0x7f0be4032420 secure :-1 s=READY pgs=29 cs=0 l=1 rev1=1 rx=0x7f0bf00096a0 tx=0x7f0bf0006b40).stop 2023-11-17T05:23:16.388 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:16.386+0000 7f0bfb59e700 1 -- 172.21.15.66:0/514962923 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f0bfc0f4920 msgr2=0x7f0bfc1091d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:23:16.388 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:16.386+0000 7f0bfb59e700 1 --2- 172.21.15.66:0/514962923 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f0bfc0f4920 0x7f0bfc1091d0 secure :-1 s=READY pgs=128 cs=0 l=1 rev1=1 rx=0x7f0bec013510 tx=0x7f0bec00b040).stop 2023-11-17T05:23:16.388 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:16.386+0000 7f0bfb59e700 1 -- 172.21.15.66:0/514962923 shutdown_connections 2023-11-17T05:23:16.388 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:16.386+0000 7f0bfb59e700 1 --2- 172.21.15.66:0/514962923 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f0be402ff60 0x7f0be4032420 unknown :-1 s=CLOSED pgs=29 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:23:16.388 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:16.386+0000 7f0bfb59e700 1 --2- 172.21.15.66:0/514962923 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f0bfc0f4920 0x7f0bfc1091d0 unknown :-1 s=CLOSED pgs=128 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:23:16.388 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:16.386+0000 7f0bfb59e700 1 -- 172.21.15.66:0/514962923 >> 172.21.15.66:0/514962923 conn(0x7f0bfc0f0120 msgr2=0x7f0bfc0f2380 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:23:16.388 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:16.387+0000 7f0bfb59e700 1 -- 172.21.15.66:0/514962923 shutdown_connections 2023-11-17T05:23:16.388 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:16.387+0000 7f0bfb59e700 1 -- 172.21.15.66:0/514962923 wait complete. 2023-11-17T05:23:16.389 INFO:teuthology.orchestra.run.smithi066.stderr:dumped monmap epoch 1 2023-11-17T05:23:16.739 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:23:16 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:23:15.175544+0000 mgr.smithi040.ogcprl (mgr.14164) 63 : cluster [DBG] pgmap v56: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:23:16.739 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:23:16 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:23:16.386557+0000 mon.smithi040 (mon.0) 141 : audit [DBG] from='client.? 172.21.15.66:0/514962923' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-11-17T05:23:17.773 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2023-11-17T05:23:17.773 DEBUG:teuthology.orchestra.run.smithi066:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.4 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ed6470e2-8508-11ee-8dbe-212e2dc638e7 -- ceph mon dump -f json 2023-11-17T05:23:18.739 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:23:18 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:23:17.175866+0000 mgr.smithi040.ogcprl (mgr.14164) 64 : cluster [DBG] pgmap v57: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:23:18.974 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:18.971+0000 7efcb33a6700 1 -- 172.21.15.66:0/2341638897 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7efcac0f5410 msgr2=0x7efcac0f5830 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:23:18.974 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:18.971+0000 7efcb33a6700 1 --2- 172.21.15.66:0/2341638897 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7efcac0f5410 0x7efcac0f5830 secure :-1 s=READY pgs=129 cs=0 l=1 rev1=1 rx=0x7efca8009fb0 tx=0x7efca8009b30).stop 2023-11-17T05:23:18.974 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:18.971+0000 7efcb33a6700 1 -- 172.21.15.66:0/2341638897 shutdown_connections 2023-11-17T05:23:18.974 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:18.971+0000 7efcb33a6700 1 --2- 172.21.15.66:0/2341638897 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7efcac0f5410 0x7efcac0f5830 unknown :-1 s=CLOSED pgs=129 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:23:18.974 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:18.971+0000 7efcb33a6700 1 -- 172.21.15.66:0/2341638897 >> 172.21.15.66:0/2341638897 conn(0x7efcac0f0b70 msgr2=0x7efcac0f2f90 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:23:18.974 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:18.971+0000 7efcb33a6700 1 -- 172.21.15.66:0/2341638897 shutdown_connections 2023-11-17T05:23:18.974 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:18.971+0000 7efcb33a6700 1 -- 172.21.15.66:0/2341638897 wait complete. 2023-11-17T05:23:18.974 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:18.972+0000 7efcb33a6700 1 Processor -- start 2023-11-17T05:23:18.974 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:18.972+0000 7efcb33a6700 1 -- start start 2023-11-17T05:23:18.974 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:18.972+0000 7efcb33a6700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7efcac0f5410 0x7efcac106710 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:23:18.974 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:18.972+0000 7efcb33a6700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7efcac106c00 con 0x7efcac0f5410 2023-11-17T05:23:18.974 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:18.972+0000 7efcb1142700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7efcac0f5410 0x7efcac106710 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:23:18.975 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:18.972+0000 7efcb1142700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7efcac0f5410 0x7efcac106710 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.66:39176/0 (socket says 172.21.15.66:39176) 2023-11-17T05:23:18.975 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:18.972+0000 7efcb1142700 1 -- 172.21.15.66:0/227707561 learned_addr learned my addr 172.21.15.66:0/227707561 (peer_addr_for_me v2:172.21.15.66:0/0) 2023-11-17T05:23:18.976 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:18.973+0000 7efcb1142700 1 -- 172.21.15.66:0/227707561 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7efca8005040 con 0x7efcac0f5410 2023-11-17T05:23:18.976 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:18.973+0000 7efcb1142700 1 --2- 172.21.15.66:0/227707561 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7efcac0f5410 0x7efcac106710 secure :-1 s=READY pgs=130 cs=0 l=1 rev1=1 rx=0x7efca8013510 tx=0x7efca800b040).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:23:18.976 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:18.973+0000 7efca1ffb700 1 -- 172.21.15.66:0/227707561 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7efca8009320 con 0x7efcac0f5410 2023-11-17T05:23:18.976 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:18.973+0000 7efcb33a6700 1 -- 172.21.15.66:0/227707561 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7efcac106e00 con 0x7efcac0f5410 2023-11-17T05:23:18.976 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:18.973+0000 7efca1ffb700 1 -- 172.21.15.66:0/227707561 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(25 keys) v1 ==== 1074+0+0 (secure 0 0 0) 0x7efca8013f00 con 0x7efcac0f5410 2023-11-17T05:23:18.976 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:18.973+0000 7efca1ffb700 1 -- 172.21.15.66:0/227707561 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7efca8014d90 con 0x7efcac0f5410 2023-11-17T05:23:18.976 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:18.973+0000 7efcb33a6700 1 -- 172.21.15.66:0/227707561 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7efcac107290 con 0x7efcac0f5410 2023-11-17T05:23:18.976 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:18.974+0000 7efca1ffb700 1 -- 172.21.15.66:0/227707561 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 17) v1 ==== 36269+0+0 (secure 0 0 0) 0x7efca80095f0 con 0x7efcac0f5410 2023-11-17T05:23:18.977 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:18.974+0000 7efcb33a6700 1 -- 172.21.15.66:0/227707561 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7efcac045ad0 con 0x7efcac0f5410 2023-11-17T05:23:18.977 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:18.974+0000 7efca1ffb700 1 --2- 172.21.15.66:0/227707561 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7efc9802ff60 0x7efc98032420 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:23:18.977 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:18.974+0000 7efca1ffb700 1 -- 172.21.15.66:0/227707561 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(5..5 src has 1..5) v4 ==== 1270+0+0 (secure 0 0 0) 0x7efca8041000 con 0x7efcac0f5410 2023-11-17T05:23:18.977 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:18.975+0000 7efcb0941700 1 --2- 172.21.15.66:0/227707561 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7efc9802ff60 0x7efc98032420 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:23:18.977 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:18.975+0000 7efcb0941700 1 --2- 172.21.15.66:0/227707561 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7efc9802ff60 0x7efc98032420 secure :-1 s=READY pgs=30 cs=0 l=1 rev1=1 rx=0x7efc9c0096a0 tx=0x7efc9c006b40).ready entity=mgr.14164 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:23:18.980 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:18.979+0000 7efca1ffb700 1 -- 172.21.15.66:0/227707561 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+152212 (secure 0 0 0) 0x7efca801d7f0 con 0x7efcac0f5410 2023-11-17T05:23:19.201 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:19.200+0000 7efcb33a6700 1 -- 172.21.15.66:0/227707561 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "mon dump", "format": "json"} v 0) v1 -- 0x7efcac0fae60 con 0x7efcac0f5410 2023-11-17T05:23:19.202 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:19.201+0000 7efca1ffb700 1 -- 172.21.15.66:0/227707561 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mon_command_ack([{"prefix": "mon dump", "format": "json"}]=0 dumped monmap epoch 1 v1) v1 ==== 95+0+690 (secure 0 0 0) 0x7efca8010230 con 0x7efcac0f5410 2023-11-17T05:23:19.202 INFO:teuthology.orchestra.run.smithi066.stdout: 2023-11-17T05:23:19.202 INFO:teuthology.orchestra.run.smithi066.stdout:{"epoch":1,"fsid":"ed6470e2-8508-11ee-8dbe-212e2dc638e7","modified":"2023-11-17T05:20:13.934038Z","created":"2023-11-17T05:20:13.934038Z","min_mon_release":16,"min_mon_release_name":"pacific","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging"],"optional":[]},"mons":[{"rank":0,"name":"smithi040","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:3300","nonce":0},{"type":"v1","addr":"172.21.15.40:6789","nonce":0}]},"addr":"172.21.15.40:6789/0","public_addr":"172.21.15.40:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2023-11-17T05:23:19.204 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:19.203+0000 7efcb33a6700 1 -- 172.21.15.66:0/227707561 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7efc9802ff60 msgr2=0x7efc98032420 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:23:19.204 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:19.203+0000 7efcb33a6700 1 --2- 172.21.15.66:0/227707561 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7efc9802ff60 0x7efc98032420 secure :-1 s=READY pgs=30 cs=0 l=1 rev1=1 rx=0x7efc9c0096a0 tx=0x7efc9c006b40).stop 2023-11-17T05:23:19.204 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:19.203+0000 7efcb33a6700 1 -- 172.21.15.66:0/227707561 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7efcac0f5410 msgr2=0x7efcac106710 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:23:19.204 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:19.203+0000 7efcb33a6700 1 --2- 172.21.15.66:0/227707561 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7efcac0f5410 0x7efcac106710 secure :-1 s=READY pgs=130 cs=0 l=1 rev1=1 rx=0x7efca8013510 tx=0x7efca800b040).stop 2023-11-17T05:23:19.204 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:19.203+0000 7efcb33a6700 1 -- 172.21.15.66:0/227707561 shutdown_connections 2023-11-17T05:23:19.204 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:19.203+0000 7efcb33a6700 1 --2- 172.21.15.66:0/227707561 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7efc9802ff60 0x7efc98032420 unknown :-1 s=CLOSED pgs=30 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:23:19.204 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:19.203+0000 7efcb33a6700 1 --2- 172.21.15.66:0/227707561 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7efcac0f5410 0x7efcac106710 unknown :-1 s=CLOSED pgs=130 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:23:19.205 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:19.203+0000 7efcb33a6700 1 -- 172.21.15.66:0/227707561 >> 172.21.15.66:0/227707561 conn(0x7efcac0f0b70 msgr2=0x7efcac05fe00 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:23:19.205 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:19.203+0000 7efcb33a6700 1 -- 172.21.15.66:0/227707561 shutdown_connections 2023-11-17T05:23:19.205 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:19.203+0000 7efcb33a6700 1 -- 172.21.15.66:0/227707561 wait complete. 2023-11-17T05:23:19.205 INFO:teuthology.orchestra.run.smithi066.stderr:dumped monmap epoch 1 2023-11-17T05:23:19.739 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:23:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:23:19.202423+0000 mon.smithi040 (mon.0) 142 : audit [DBG] from='client.? 172.21.15.66:0/227707561' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-11-17T05:23:20.596 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2023-11-17T05:23:20.597 DEBUG:teuthology.orchestra.run.smithi066:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.4 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ed6470e2-8508-11ee-8dbe-212e2dc638e7 -- ceph mon dump -f json 2023-11-17T05:23:20.738 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:23:20 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:23:19.176213+0000 mgr.smithi040.ogcprl (mgr.14164) 65 : cluster [DBG] pgmap v58: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:23:21.818 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:21.816+0000 7fbca5ffb700 1 -- 172.21.15.66:0/1644212142 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7fbca00138a0 con 0x7fbca80f5410 2023-11-17T05:23:21.818 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:21.817+0000 7fbcad36c700 1 -- 172.21.15.66:0/1644212142 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fbca80f5410 msgr2=0x7fbca80f5830 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:23:21.818 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:21.817+0000 7fbcad36c700 1 --2- 172.21.15.66:0/1644212142 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fbca80f5410 0x7fbca80f5830 secure :-1 s=READY pgs=131 cs=0 l=1 rev1=1 rx=0x7fbca0009fb0 tx=0x7fbca0009b30).stop 2023-11-17T05:23:21.818 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:21.817+0000 7fbcad36c700 1 -- 172.21.15.66:0/1644212142 shutdown_connections 2023-11-17T05:23:21.818 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:21.817+0000 7fbcad36c700 1 --2- 172.21.15.66:0/1644212142 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fbca80f5410 0x7fbca80f5830 unknown :-1 s=CLOSED pgs=131 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:23:21.818 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:21.817+0000 7fbcad36c700 1 -- 172.21.15.66:0/1644212142 >> 172.21.15.66:0/1644212142 conn(0x7fbca80f0b70 msgr2=0x7fbca80f2f90 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:23:21.818 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:21.817+0000 7fbcad36c700 1 -- 172.21.15.66:0/1644212142 shutdown_connections 2023-11-17T05:23:21.818 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:21.817+0000 7fbcad36c700 1 -- 172.21.15.66:0/1644212142 wait complete. 2023-11-17T05:23:21.820 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:21.817+0000 7fbcad36c700 1 Processor -- start 2023-11-17T05:23:21.820 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:21.817+0000 7fbcad36c700 1 -- start start 2023-11-17T05:23:21.820 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:21.817+0000 7fbcad36c700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fbca80f5410 0x7fbca8108910 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:23:21.820 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:21.817+0000 7fbcad36c700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fbca8108e00 con 0x7fbca80f5410 2023-11-17T05:23:21.820 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:21.818+0000 7fbca6ffd700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fbca80f5410 0x7fbca8108910 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:23:21.820 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:21.818+0000 7fbca6ffd700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fbca80f5410 0x7fbca8108910 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.66:39186/0 (socket says 172.21.15.66:39186) 2023-11-17T05:23:21.820 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:21.818+0000 7fbca6ffd700 1 -- 172.21.15.66:0/1625306907 learned_addr learned my addr 172.21.15.66:0/1625306907 (peer_addr_for_me v2:172.21.15.66:0/0) 2023-11-17T05:23:21.820 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:21.818+0000 7fbca6ffd700 1 -- 172.21.15.66:0/1625306907 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fbca0005040 con 0x7fbca80f5410 2023-11-17T05:23:21.820 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:21.818+0000 7fbca6ffd700 1 --2- 172.21.15.66:0/1625306907 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fbca80f5410 0x7fbca8108910 secure :-1 s=READY pgs=132 cs=0 l=1 rev1=1 rx=0x7fbca0013510 tx=0x7fbca000b040).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:23:21.821 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:21.818+0000 7fbc9f7fe700 1 -- 172.21.15.66:0/1625306907 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7fbca0009320 con 0x7fbca80f5410 2023-11-17T05:23:21.821 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:21.818+0000 7fbcad36c700 1 -- 172.21.15.66:0/1625306907 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fbca8109000 con 0x7fbca80f5410 2023-11-17T05:23:21.821 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:21.818+0000 7fbcad36c700 1 -- 172.21.15.66:0/1625306907 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fbca8109490 con 0x7fbca80f5410 2023-11-17T05:23:21.821 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:21.819+0000 7fbc9f7fe700 1 -- 172.21.15.66:0/1625306907 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(25 keys) v1 ==== 1074+0+0 (secure 0 0 0) 0x7fbca0013f00 con 0x7fbca80f5410 2023-11-17T05:23:21.821 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:21.819+0000 7fbc9f7fe700 1 -- 172.21.15.66:0/1625306907 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7fbca0014ce0 con 0x7fbca80f5410 2023-11-17T05:23:21.822 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:21.819+0000 7fbc9f7fe700 1 -- 172.21.15.66:0/1625306907 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 17) v1 ==== 36269+0+0 (secure 0 0 0) 0x7fbca0014420 con 0x7fbca80f5410 2023-11-17T05:23:21.822 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:21.819+0000 7fbcad36c700 1 -- 172.21.15.66:0/1625306907 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fbc8c004fa0 con 0x7fbca80f5410 2023-11-17T05:23:21.822 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:21.820+0000 7fbc9f7fe700 1 --2- 172.21.15.66:0/1625306907 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fbc9402ff60 0x7fbc94032420 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:23:21.822 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:21.820+0000 7fbc9f7fe700 1 -- 172.21.15.66:0/1625306907 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(5..5 src has 1..5) v4 ==== 1270+0+0 (secure 0 0 0) 0x7fbca0041140 con 0x7fbca80f5410 2023-11-17T05:23:21.822 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:21.820+0000 7fbca67fc700 1 --2- 172.21.15.66:0/1625306907 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fbc9402ff60 0x7fbc94032420 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:23:21.822 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:21.820+0000 7fbca67fc700 1 --2- 172.21.15.66:0/1625306907 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fbc9402ff60 0x7fbc94032420 secure :-1 s=READY pgs=31 cs=0 l=1 rev1=1 rx=0x7fbc900096a0 tx=0x7fbc90006b40).ready entity=mgr.14164 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:23:21.826 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:21.825+0000 7fbc9f7fe700 1 -- 172.21.15.66:0/1625306907 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+152212 (secure 0 0 0) 0x7fbca0012930 con 0x7fbca80f5410 2023-11-17T05:23:22.044 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:22.042+0000 7fbcad36c700 1 -- 172.21.15.66:0/1625306907 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "mon dump", "format": "json"} v 0) v1 -- 0x7fbc8c005670 con 0x7fbca80f5410 2023-11-17T05:23:22.044 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:22.043+0000 7fbc9f7fe700 1 -- 172.21.15.66:0/1625306907 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mon_command_ack([{"prefix": "mon dump", "format": "json"}]=0 dumped monmap epoch 1 v1) v1 ==== 95+0+690 (secure 0 0 0) 0x7fbca0012930 con 0x7fbca80f5410 2023-11-17T05:23:22.045 INFO:teuthology.orchestra.run.smithi066.stdout: 2023-11-17T05:23:22.045 INFO:teuthology.orchestra.run.smithi066.stdout:{"epoch":1,"fsid":"ed6470e2-8508-11ee-8dbe-212e2dc638e7","modified":"2023-11-17T05:20:13.934038Z","created":"2023-11-17T05:20:13.934038Z","min_mon_release":16,"min_mon_release_name":"pacific","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging"],"optional":[]},"mons":[{"rank":0,"name":"smithi040","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:3300","nonce":0},{"type":"v1","addr":"172.21.15.40:6789","nonce":0}]},"addr":"172.21.15.40:6789/0","public_addr":"172.21.15.40:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2023-11-17T05:23:22.046 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:22.044+0000 7fbcad36c700 1 -- 172.21.15.66:0/1625306907 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fbc9402ff60 msgr2=0x7fbc94032420 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:23:22.046 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:22.044+0000 7fbcad36c700 1 --2- 172.21.15.66:0/1625306907 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fbc9402ff60 0x7fbc94032420 secure :-1 s=READY pgs=31 cs=0 l=1 rev1=1 rx=0x7fbc900096a0 tx=0x7fbc90006b40).stop 2023-11-17T05:23:22.046 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:22.045+0000 7fbcad36c700 1 -- 172.21.15.66:0/1625306907 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fbca80f5410 msgr2=0x7fbca8108910 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:23:22.046 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:22.045+0000 7fbcad36c700 1 --2- 172.21.15.66:0/1625306907 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fbca80f5410 0x7fbca8108910 secure :-1 s=READY pgs=132 cs=0 l=1 rev1=1 rx=0x7fbca0013510 tx=0x7fbca000b040).stop 2023-11-17T05:23:22.046 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:22.045+0000 7fbcad36c700 1 -- 172.21.15.66:0/1625306907 shutdown_connections 2023-11-17T05:23:22.046 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:22.045+0000 7fbcad36c700 1 --2- 172.21.15.66:0/1625306907 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fbc9402ff60 0x7fbc94032420 unknown :-1 s=CLOSED pgs=31 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:23:22.046 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:22.045+0000 7fbcad36c700 1 --2- 172.21.15.66:0/1625306907 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fbca80f5410 0x7fbca8108910 unknown :-1 s=CLOSED pgs=132 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:23:22.046 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:22.045+0000 7fbcad36c700 1 -- 172.21.15.66:0/1625306907 >> 172.21.15.66:0/1625306907 conn(0x7fbca80f0b70 msgr2=0x7fbca80f2f60 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:23:22.047 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:22.045+0000 7fbcad36c700 1 -- 172.21.15.66:0/1625306907 shutdown_connections 2023-11-17T05:23:22.047 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:22.045+0000 7fbcad36c700 1 -- 172.21.15.66:0/1625306907 wait complete. 2023-11-17T05:23:22.047 INFO:teuthology.orchestra.run.smithi066.stderr:dumped monmap epoch 1 2023-11-17T05:23:22.738 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:23:22 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:23:21.176468+0000 mgr.smithi040.ogcprl (mgr.14164) 66 : cluster [DBG] pgmap v59: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:23:22.739 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:23:22 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:23:22.044744+0000 mon.smithi040 (mon.0) 143 : audit [DBG] from='client.? 172.21.15.66:0/1625306907' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-11-17T05:23:23.456 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2023-11-17T05:23:23.457 DEBUG:teuthology.orchestra.run.smithi066:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.4 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ed6470e2-8508-11ee-8dbe-212e2dc638e7 -- ceph mon dump -f json 2023-11-17T05:23:24.634 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:24.631+0000 7f1c2bd36700 1 -- 172.21.15.66:0/607511378 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f1c240f6510 msgr2=0x7f1c240f6930 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:23:24.634 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:24.631+0000 7f1c2bd36700 1 --2- 172.21.15.66:0/607511378 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f1c240f6510 0x7f1c240f6930 secure :-1 s=READY pgs=133 cs=0 l=1 rev1=1 rx=0x7f1c20009fb0 tx=0x7f1c20009b30).stop 2023-11-17T05:23:24.634 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:24.631+0000 7f1c2bd36700 1 -- 172.21.15.66:0/607511378 shutdown_connections 2023-11-17T05:23:24.634 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:24.631+0000 7f1c2bd36700 1 --2- 172.21.15.66:0/607511378 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f1c240f6510 0x7f1c240f6930 unknown :-1 s=CLOSED pgs=133 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:23:24.634 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:24.631+0000 7f1c2bd36700 1 -- 172.21.15.66:0/607511378 >> 172.21.15.66:0/607511378 conn(0x7f1c240f1c70 msgr2=0x7f1c240f4090 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:23:24.634 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:24.631+0000 7f1c2bd36700 1 -- 172.21.15.66:0/607511378 shutdown_connections 2023-11-17T05:23:24.634 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:24.631+0000 7f1c2bd36700 1 -- 172.21.15.66:0/607511378 wait complete. 2023-11-17T05:23:24.634 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:24.631+0000 7f1c2bd36700 1 Processor -- start 2023-11-17T05:23:24.634 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:24.631+0000 7f1c2bd36700 1 -- start start 2023-11-17T05:23:24.634 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:24.632+0000 7f1c2bd36700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f1c240f6510 0x7f1c24106730 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:23:24.635 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:24.632+0000 7f1c2bd36700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f1c24106c20 con 0x7f1c240f6510 2023-11-17T05:23:24.635 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:24.632+0000 7f1c29ad2700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f1c240f6510 0x7f1c24106730 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:23:24.635 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:24.632+0000 7f1c29ad2700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f1c240f6510 0x7f1c24106730 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.66:54128/0 (socket says 172.21.15.66:54128) 2023-11-17T05:23:24.635 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:24.632+0000 7f1c29ad2700 1 -- 172.21.15.66:0/475885309 learned_addr learned my addr 172.21.15.66:0/475885309 (peer_addr_for_me v2:172.21.15.66:0/0) 2023-11-17T05:23:24.635 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:24.632+0000 7f1c29ad2700 1 -- 172.21.15.66:0/475885309 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f1c20005040 con 0x7f1c240f6510 2023-11-17T05:23:24.635 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:24.632+0000 7f1c29ad2700 1 --2- 172.21.15.66:0/475885309 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f1c240f6510 0x7f1c24106730 secure :-1 s=READY pgs=134 cs=0 l=1 rev1=1 rx=0x7f1c20013510 tx=0x7f1c2000b040).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:23:24.635 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:24.632+0000 7f1c1a7fc700 1 -- 172.21.15.66:0/475885309 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f1c20009320 con 0x7f1c240f6510 2023-11-17T05:23:24.635 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:24.632+0000 7f1c2bd36700 1 -- 172.21.15.66:0/475885309 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f1c24106e20 con 0x7f1c240f6510 2023-11-17T05:23:24.635 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:24.632+0000 7f1c1a7fc700 1 -- 172.21.15.66:0/475885309 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(25 keys) v1 ==== 1074+0+0 (secure 0 0 0) 0x7f1c20013f00 con 0x7f1c240f6510 2023-11-17T05:23:24.635 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:24.632+0000 7f1c1a7fc700 1 -- 172.21.15.66:0/475885309 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f1c20014d90 con 0x7f1c240f6510 2023-11-17T05:23:24.635 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:24.632+0000 7f1c2bd36700 1 -- 172.21.15.66:0/475885309 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f1c241072b0 con 0x7f1c240f6510 2023-11-17T05:23:24.636 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:24.633+0000 7f1c1a7fc700 1 -- 172.21.15.66:0/475885309 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 17) v1 ==== 36269+0+0 (secure 0 0 0) 0x7f1c200263a0 con 0x7f1c240f6510 2023-11-17T05:23:24.636 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:24.633+0000 7f1c2bd36700 1 -- 172.21.15.66:0/475885309 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f1c24045ad0 con 0x7f1c240f6510 2023-11-17T05:23:24.636 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:24.633+0000 7f1c1a7fc700 1 --2- 172.21.15.66:0/475885309 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f1c1002ff60 0x7f1c10032420 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:23:24.636 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:24.633+0000 7f1c1a7fc700 1 -- 172.21.15.66:0/475885309 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(5..5 src has 1..5) v4 ==== 1270+0+0 (secure 0 0 0) 0x7f1c200412f0 con 0x7f1c240f6510 2023-11-17T05:23:24.636 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:24.634+0000 7f1c292d1700 1 --2- 172.21.15.66:0/475885309 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f1c1002ff60 0x7f1c10032420 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:23:24.637 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:24.634+0000 7f1c292d1700 1 --2- 172.21.15.66:0/475885309 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f1c1002ff60 0x7f1c10032420 secure :-1 s=READY pgs=32 cs=0 l=1 rev1=1 rx=0x7f1c140096a0 tx=0x7f1c14006b40).ready entity=mgr.14164 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:23:24.640 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:24.638+0000 7f1c1a7fc700 1 -- 172.21.15.66:0/475885309 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+152212 (secure 0 0 0) 0x7f1c2001d3d0 con 0x7f1c240f6510 2023-11-17T05:23:24.739 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:23:24 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:23:23.176837+0000 mgr.smithi040.ogcprl (mgr.14164) 67 : cluster [DBG] pgmap v60: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:23:24.861 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:24.860+0000 7f1c2bd36700 1 -- 172.21.15.66:0/475885309 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "mon dump", "format": "json"} v 0) v1 -- 0x7f1c240fbf60 con 0x7f1c240f6510 2023-11-17T05:23:24.862 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:24.862+0000 7f1c1a7fc700 1 -- 172.21.15.66:0/475885309 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mon_command_ack([{"prefix": "mon dump", "format": "json"}]=0 dumped monmap epoch 1 v1) v1 ==== 95+0+690 (secure 0 0 0) 0x7f1c2000f340 con 0x7f1c240f6510 2023-11-17T05:23:24.862 INFO:teuthology.orchestra.run.smithi066.stdout: 2023-11-17T05:23:24.862 INFO:teuthology.orchestra.run.smithi066.stdout:{"epoch":1,"fsid":"ed6470e2-8508-11ee-8dbe-212e2dc638e7","modified":"2023-11-17T05:20:13.934038Z","created":"2023-11-17T05:20:13.934038Z","min_mon_release":16,"min_mon_release_name":"pacific","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging"],"optional":[]},"mons":[{"rank":0,"name":"smithi040","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:3300","nonce":0},{"type":"v1","addr":"172.21.15.40:6789","nonce":0}]},"addr":"172.21.15.40:6789/0","public_addr":"172.21.15.40:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2023-11-17T05:23:24.864 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:24.862+0000 7f1c2bd36700 1 -- 172.21.15.66:0/475885309 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f1c1002ff60 msgr2=0x7f1c10032420 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:23:24.864 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:24.862+0000 7f1c2bd36700 1 --2- 172.21.15.66:0/475885309 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f1c1002ff60 0x7f1c10032420 secure :-1 s=READY pgs=32 cs=0 l=1 rev1=1 rx=0x7f1c140096a0 tx=0x7f1c14006b40).stop 2023-11-17T05:23:24.864 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:24.862+0000 7f1c2bd36700 1 -- 172.21.15.66:0/475885309 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f1c240f6510 msgr2=0x7f1c24106730 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:23:24.864 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:24.862+0000 7f1c2bd36700 1 --2- 172.21.15.66:0/475885309 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f1c240f6510 0x7f1c24106730 secure :-1 s=READY pgs=134 cs=0 l=1 rev1=1 rx=0x7f1c20013510 tx=0x7f1c2000b040).stop 2023-11-17T05:23:24.864 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:24.862+0000 7f1c2bd36700 1 -- 172.21.15.66:0/475885309 shutdown_connections 2023-11-17T05:23:24.864 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:24.862+0000 7f1c2bd36700 1 --2- 172.21.15.66:0/475885309 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f1c1002ff60 0x7f1c10032420 unknown :-1 s=CLOSED pgs=32 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:23:24.864 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:24.862+0000 7f1c2bd36700 1 --2- 172.21.15.66:0/475885309 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f1c240f6510 0x7f1c24106730 unknown :-1 s=CLOSED pgs=134 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:23:24.877 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:24.862+0000 7f1c2bd36700 1 -- 172.21.15.66:0/475885309 >> 172.21.15.66:0/475885309 conn(0x7f1c240f1c70 msgr2=0x7f1c240f2760 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:23:24.877 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:24.862+0000 7f1c2bd36700 1 -- 172.21.15.66:0/475885309 shutdown_connections 2023-11-17T05:23:24.878 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:24.862+0000 7f1c2bd36700 1 -- 172.21.15.66:0/475885309 wait complete. 2023-11-17T05:23:24.878 INFO:teuthology.orchestra.run.smithi066.stderr:dumped monmap epoch 1 2023-11-17T05:23:25.739 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:23:25 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:23:24.862495+0000 mon.smithi040 (mon.0) 144 : audit [DBG] from='client.? 172.21.15.66:0/475885309' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-11-17T05:23:26.247 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2023-11-17T05:23:26.248 DEBUG:teuthology.orchestra.run.smithi066:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.4 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ed6470e2-8508-11ee-8dbe-212e2dc638e7 -- ceph mon dump -f json 2023-11-17T05:23:26.739 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:23:26 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:23:25.177169+0000 mgr.smithi040.ogcprl (mgr.14164) 68 : cluster [DBG] pgmap v61: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:23:27.437 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:27.436+0000 7fd80a551700 1 -- 172.21.15.66:0/2101545714 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fd804060220 msgr2=0x7fd8040f9150 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:23:27.437 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:27.436+0000 7fd80a551700 1 --2- 172.21.15.66:0/2101545714 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fd804060220 0x7fd8040f9150 secure :-1 s=READY pgs=135 cs=0 l=1 rev1=1 rx=0x7fd7f4009fb0 tx=0x7fd7f4009b30).stop 2023-11-17T05:23:27.437 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:27.436+0000 7fd80a551700 1 -- 172.21.15.66:0/2101545714 shutdown_connections 2023-11-17T05:23:27.437 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:27.436+0000 7fd80a551700 1 --2- 172.21.15.66:0/2101545714 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fd804060220 0x7fd8040f9150 unknown :-1 s=CLOSED pgs=135 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:23:27.437 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:27.436+0000 7fd80a551700 1 -- 172.21.15.66:0/2101545714 >> 172.21.15.66:0/2101545714 conn(0x7fd8040f0b70 msgr2=0x7fd8040f2f90 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:23:27.438 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:27.436+0000 7fd80a551700 1 -- 172.21.15.66:0/2101545714 shutdown_connections 2023-11-17T05:23:27.438 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:27.436+0000 7fd80a551700 1 -- 172.21.15.66:0/2101545714 wait complete. 2023-11-17T05:23:27.439 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:27.436+0000 7fd80a551700 1 Processor -- start 2023-11-17T05:23:27.439 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:27.437+0000 7fd80a551700 1 -- start start 2023-11-17T05:23:27.439 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:27.437+0000 7fd80a551700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fd804060220 0x7fd8041066e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:23:27.439 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:27.437+0000 7fd80a551700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fd804106bd0 con 0x7fd804060220 2023-11-17T05:23:27.440 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:27.437+0000 7fd803fff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fd804060220 0x7fd8041066e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:23:27.440 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:27.437+0000 7fd803fff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fd804060220 0x7fd8041066e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.66:54152/0 (socket says 172.21.15.66:54152) 2023-11-17T05:23:27.440 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:27.437+0000 7fd803fff700 1 -- 172.21.15.66:0/3867466055 learned_addr learned my addr 172.21.15.66:0/3867466055 (peer_addr_for_me v2:172.21.15.66:0/0) 2023-11-17T05:23:27.440 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:27.437+0000 7fd803fff700 1 -- 172.21.15.66:0/3867466055 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fd7f4005040 con 0x7fd804060220 2023-11-17T05:23:27.440 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:27.437+0000 7fd803fff700 1 --2- 172.21.15.66:0/3867466055 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fd804060220 0x7fd8041066e0 secure :-1 s=READY pgs=136 cs=0 l=1 rev1=1 rx=0x7fd7f4013510 tx=0x7fd7f400b040).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:23:27.440 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:27.438+0000 7fd800ff9700 1 -- 172.21.15.66:0/3867466055 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7fd7f4009320 con 0x7fd804060220 2023-11-17T05:23:27.440 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:27.438+0000 7fd80a551700 1 -- 172.21.15.66:0/3867466055 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fd804106dd0 con 0x7fd804060220 2023-11-17T05:23:27.440 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:27.438+0000 7fd800ff9700 1 -- 172.21.15.66:0/3867466055 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(25 keys) v1 ==== 1074+0+0 (secure 0 0 0) 0x7fd7f4013f00 con 0x7fd804060220 2023-11-17T05:23:27.440 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:27.438+0000 7fd800ff9700 1 -- 172.21.15.66:0/3867466055 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7fd7f4014dc0 con 0x7fd804060220 2023-11-17T05:23:27.440 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:27.438+0000 7fd80a551700 1 -- 172.21.15.66:0/3867466055 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fd804107260 con 0x7fd804060220 2023-11-17T05:23:27.441 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:27.438+0000 7fd800ff9700 1 -- 172.21.15.66:0/3867466055 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 17) v1 ==== 36269+0+0 (secure 0 0 0) 0x7fd7f40095f0 con 0x7fd804060220 2023-11-17T05:23:27.441 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:27.439+0000 7fd80a551700 1 -- 172.21.15.66:0/3867466055 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fd804045ad0 con 0x7fd804060220 2023-11-17T05:23:27.441 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:27.439+0000 7fd800ff9700 1 --2- 172.21.15.66:0/3867466055 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fd7ec02ff60 0x7fd7ec032420 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:23:27.441 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:27.439+0000 7fd800ff9700 1 -- 172.21.15.66:0/3867466055 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(5..5 src has 1..5) v4 ==== 1270+0+0 (secure 0 0 0) 0x7fd7f40410f0 con 0x7fd804060220 2023-11-17T05:23:27.442 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:27.439+0000 7fd8037fe700 1 --2- 172.21.15.66:0/3867466055 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fd7ec02ff60 0x7fd7ec032420 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:23:27.442 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:27.439+0000 7fd8037fe700 1 --2- 172.21.15.66:0/3867466055 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fd7ec02ff60 0x7fd7ec032420 secure :-1 s=READY pgs=33 cs=0 l=1 rev1=1 rx=0x7fd7f80096a0 tx=0x7fd7f8006b40).ready entity=mgr.14164 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:23:27.445 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:27.444+0000 7fd800ff9700 1 -- 172.21.15.66:0/3867466055 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+152212 (secure 0 0 0) 0x7fd7f4014420 con 0x7fd804060220 2023-11-17T05:23:27.670 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:27.668+0000 7fd80a551700 1 -- 172.21.15.66:0/3867466055 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "mon dump", "format": "json"} v 0) v1 -- 0x7fd804108110 con 0x7fd804060220 2023-11-17T05:23:27.670 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:27.669+0000 7fd800ff9700 1 -- 172.21.15.66:0/3867466055 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mon_command_ack([{"prefix": "mon dump", "format": "json"}]=0 dumped monmap epoch 1 v1) v1 ==== 95+0+690 (secure 0 0 0) 0x7fd7f4010230 con 0x7fd804060220 2023-11-17T05:23:27.671 INFO:teuthology.orchestra.run.smithi066.stdout: 2023-11-17T05:23:27.671 INFO:teuthology.orchestra.run.smithi066.stdout:{"epoch":1,"fsid":"ed6470e2-8508-11ee-8dbe-212e2dc638e7","modified":"2023-11-17T05:20:13.934038Z","created":"2023-11-17T05:20:13.934038Z","min_mon_release":16,"min_mon_release_name":"pacific","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging"],"optional":[]},"mons":[{"rank":0,"name":"smithi040","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:3300","nonce":0},{"type":"v1","addr":"172.21.15.40:6789","nonce":0}]},"addr":"172.21.15.40:6789/0","public_addr":"172.21.15.40:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2023-11-17T05:23:27.672 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:27.670+0000 7fd80a551700 1 -- 172.21.15.66:0/3867466055 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fd7ec02ff60 msgr2=0x7fd7ec032420 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:23:27.673 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:27.670+0000 7fd80a551700 1 --2- 172.21.15.66:0/3867466055 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fd7ec02ff60 0x7fd7ec032420 secure :-1 s=READY pgs=33 cs=0 l=1 rev1=1 rx=0x7fd7f80096a0 tx=0x7fd7f8006b40).stop 2023-11-17T05:23:27.673 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:27.670+0000 7fd80a551700 1 -- 172.21.15.66:0/3867466055 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fd804060220 msgr2=0x7fd8041066e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:23:27.673 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:27.670+0000 7fd80a551700 1 --2- 172.21.15.66:0/3867466055 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fd804060220 0x7fd8041066e0 secure :-1 s=READY pgs=136 cs=0 l=1 rev1=1 rx=0x7fd7f4013510 tx=0x7fd7f400b040).stop 2023-11-17T05:23:27.673 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:27.670+0000 7fd80a551700 1 -- 172.21.15.66:0/3867466055 shutdown_connections 2023-11-17T05:23:27.673 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:27.670+0000 7fd80a551700 1 --2- 172.21.15.66:0/3867466055 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fd7ec02ff60 0x7fd7ec032420 unknown :-1 s=CLOSED pgs=33 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:23:27.673 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:27.670+0000 7fd80a551700 1 --2- 172.21.15.66:0/3867466055 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fd804060220 0x7fd8041066e0 unknown :-1 s=CLOSED pgs=136 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:23:27.673 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:27.670+0000 7fd80a551700 1 -- 172.21.15.66:0/3867466055 >> 172.21.15.66:0/3867466055 conn(0x7fd8040f0b70 msgr2=0x7fd8040f1660 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:23:27.673 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:27.671+0000 7fd80a551700 1 -- 172.21.15.66:0/3867466055 shutdown_connections 2023-11-17T05:23:27.673 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:27.671+0000 7fd80a551700 1 -- 172.21.15.66:0/3867466055 wait complete. 2023-11-17T05:23:27.673 INFO:teuthology.orchestra.run.smithi066.stderr:dumped monmap epoch 1 2023-11-17T05:23:28.739 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:23:28 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:23:27.177505+0000 mgr.smithi040.ogcprl (mgr.14164) 69 : cluster [DBG] pgmap v62: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:23:28.739 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:23:28 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:23:27.670359+0000 mon.smithi040 (mon.0) 145 : audit [DBG] from='client.? 172.21.15.66:0/3867466055' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-11-17T05:23:29.042 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2023-11-17T05:23:29.042 DEBUG:teuthology.orchestra.run.smithi066:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.4 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ed6470e2-8508-11ee-8dbe-212e2dc638e7 -- ceph mon dump -f json 2023-11-17T05:23:30.275 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:30.272+0000 7fc79ad08700 1 -- 172.21.15.66:0/34910911 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fc7940f47c0 msgr2=0x7fc7940f4be0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:23:30.275 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:30.272+0000 7fc79ad08700 1 --2- 172.21.15.66:0/34910911 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fc7940f47c0 0x7fc7940f4be0 secure :-1 s=READY pgs=137 cs=0 l=1 rev1=1 rx=0x7fc788009fb0 tx=0x7fc788009b30).stop 2023-11-17T05:23:30.275 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:30.272+0000 7fc79ad08700 1 -- 172.21.15.66:0/34910911 shutdown_connections 2023-11-17T05:23:30.275 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:30.272+0000 7fc79ad08700 1 --2- 172.21.15.66:0/34910911 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fc7940f47c0 0x7fc7940f4be0 unknown :-1 s=CLOSED pgs=137 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:23:30.276 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:30.272+0000 7fc79ad08700 1 -- 172.21.15.66:0/34910911 >> 172.21.15.66:0/34910911 conn(0x7fc7940f02e0 msgr2=0x7fc7940f2700 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:23:30.276 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:30.272+0000 7fc79ad08700 1 -- 172.21.15.66:0/34910911 shutdown_connections 2023-11-17T05:23:30.276 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:30.272+0000 7fc79ad08700 1 -- 172.21.15.66:0/34910911 wait complete. 2023-11-17T05:23:30.276 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:30.272+0000 7fc79ad08700 1 Processor -- start 2023-11-17T05:23:30.276 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:30.272+0000 7fc79ad08700 1 -- start start 2023-11-17T05:23:30.276 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:30.273+0000 7fc79ad08700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fc7940f47c0 0x7fc794106700 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:23:30.276 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:30.273+0000 7fc79ad08700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fc794106bf0 con 0x7fc7940f47c0 2023-11-17T05:23:30.276 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:30.273+0000 7fc798aa4700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fc7940f47c0 0x7fc794106700 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:23:30.276 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:30.273+0000 7fc798aa4700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fc7940f47c0 0x7fc794106700 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.66:54168/0 (socket says 172.21.15.66:54168) 2023-11-17T05:23:30.276 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:30.273+0000 7fc798aa4700 1 -- 172.21.15.66:0/2029749841 learned_addr learned my addr 172.21.15.66:0/2029749841 (peer_addr_for_me v2:172.21.15.66:0/0) 2023-11-17T05:23:30.276 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:30.273+0000 7fc798aa4700 1 -- 172.21.15.66:0/2029749841 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fc788005040 con 0x7fc7940f47c0 2023-11-17T05:23:30.276 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:30.273+0000 7fc798aa4700 1 --2- 172.21.15.66:0/2029749841 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fc7940f47c0 0x7fc794106700 secure :-1 s=READY pgs=138 cs=0 l=1 rev1=1 rx=0x7fc788013510 tx=0x7fc78800b040).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:23:30.276 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:30.274+0000 7fc7917fa700 1 -- 172.21.15.66:0/2029749841 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7fc788009320 con 0x7fc7940f47c0 2023-11-17T05:23:30.277 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:30.274+0000 7fc7917fa700 1 -- 172.21.15.66:0/2029749841 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(25 keys) v1 ==== 1074+0+0 (secure 0 0 0) 0x7fc788013f00 con 0x7fc7940f47c0 2023-11-17T05:23:30.277 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:30.274+0000 7fc79ad08700 1 -- 172.21.15.66:0/2029749841 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fc794106df0 con 0x7fc7940f47c0 2023-11-17T05:23:30.277 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:30.274+0000 7fc7917fa700 1 -- 172.21.15.66:0/2029749841 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7fc788014d90 con 0x7fc7940f47c0 2023-11-17T05:23:30.277 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:30.274+0000 7fc79ad08700 1 -- 172.21.15.66:0/2029749841 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fc794107280 con 0x7fc7940f47c0 2023-11-17T05:23:30.278 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:30.274+0000 7fc7917fa700 1 -- 172.21.15.66:0/2029749841 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 17) v1 ==== 36269+0+0 (secure 0 0 0) 0x7fc7880263a0 con 0x7fc7940f47c0 2023-11-17T05:23:30.278 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:30.275+0000 7fc7917fa700 1 --2- 172.21.15.66:0/2029749841 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fc78402ffb0 0x7fc784032470 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:23:30.278 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:30.275+0000 7fc7917fa700 1 -- 172.21.15.66:0/2029749841 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(5..5 src has 1..5) v4 ==== 1270+0+0 (secure 0 0 0) 0x7fc788040fe0 con 0x7fc7940f47c0 2023-11-17T05:23:30.278 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:30.275+0000 7fc79ad08700 1 -- 172.21.15.66:0/2029749841 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fc794045ad0 con 0x7fc7940f47c0 2023-11-17T05:23:30.278 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:30.275+0000 7fc793fff700 1 --2- 172.21.15.66:0/2029749841 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fc78402ffb0 0x7fc784032470 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:23:30.278 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:30.275+0000 7fc793fff700 1 --2- 172.21.15.66:0/2029749841 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fc78402ffb0 0x7fc784032470 secure :-1 s=READY pgs=34 cs=0 l=1 rev1=1 rx=0x7fc7800096a0 tx=0x7fc780006b40).ready entity=mgr.14164 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:23:30.282 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:30.280+0000 7fc7917fa700 1 -- 172.21.15.66:0/2029749841 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+152212 (secure 0 0 0) 0x7fc788012920 con 0x7fc7940f47c0 2023-11-17T05:23:30.503 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:30.500+0000 7fc79ad08700 1 -- 172.21.15.66:0/2029749841 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "mon dump", "format": "json"} v 0) v1 -- 0x7fc7940fbf60 con 0x7fc7940f47c0 2023-11-17T05:23:30.504 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:30.501+0000 7fc7917fa700 1 -- 172.21.15.66:0/2029749841 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mon_command_ack([{"prefix": "mon dump", "format": "json"}]=0 dumped monmap epoch 1 v1) v1 ==== 95+0+690 (secure 0 0 0) 0x7fc78802b020 con 0x7fc7940f47c0 2023-11-17T05:23:30.505 INFO:teuthology.orchestra.run.smithi066.stdout: 2023-11-17T05:23:30.506 INFO:teuthology.orchestra.run.smithi066.stdout:{"epoch":1,"fsid":"ed6470e2-8508-11ee-8dbe-212e2dc638e7","modified":"2023-11-17T05:20:13.934038Z","created":"2023-11-17T05:20:13.934038Z","min_mon_release":16,"min_mon_release_name":"pacific","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging"],"optional":[]},"mons":[{"rank":0,"name":"smithi040","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:3300","nonce":0},{"type":"v1","addr":"172.21.15.40:6789","nonce":0}]},"addr":"172.21.15.40:6789/0","public_addr":"172.21.15.40:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2023-11-17T05:23:30.511 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:30.502+0000 7fc79ad08700 1 -- 172.21.15.66:0/2029749841 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fc78402ffb0 msgr2=0x7fc784032470 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:23:30.512 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:30.502+0000 7fc79ad08700 1 --2- 172.21.15.66:0/2029749841 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fc78402ffb0 0x7fc784032470 secure :-1 s=READY pgs=34 cs=0 l=1 rev1=1 rx=0x7fc7800096a0 tx=0x7fc780006b40).stop 2023-11-17T05:23:30.512 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:30.502+0000 7fc79ad08700 1 -- 172.21.15.66:0/2029749841 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fc7940f47c0 msgr2=0x7fc794106700 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:23:30.513 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:30.502+0000 7fc79ad08700 1 --2- 172.21.15.66:0/2029749841 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fc7940f47c0 0x7fc794106700 secure :-1 s=READY pgs=138 cs=0 l=1 rev1=1 rx=0x7fc788013510 tx=0x7fc78800b040).stop 2023-11-17T05:23:30.513 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:30.503+0000 7fc79ad08700 1 -- 172.21.15.66:0/2029749841 shutdown_connections 2023-11-17T05:23:30.513 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:30.503+0000 7fc79ad08700 1 --2- 172.21.15.66:0/2029749841 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fc78402ffb0 0x7fc784032470 unknown :-1 s=CLOSED pgs=34 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:23:30.513 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:30.503+0000 7fc79ad08700 1 --2- 172.21.15.66:0/2029749841 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fc7940f47c0 0x7fc794106700 unknown :-1 s=CLOSED pgs=138 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:23:30.513 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:30.503+0000 7fc79ad08700 1 -- 172.21.15.66:0/2029749841 >> 172.21.15.66:0/2029749841 conn(0x7fc7940f02e0 msgr2=0x7fc7940f0db0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:23:30.513 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:30.503+0000 7fc79ad08700 1 -- 172.21.15.66:0/2029749841 shutdown_connections 2023-11-17T05:23:30.513 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:30.503+0000 7fc79ad08700 1 -- 172.21.15.66:0/2029749841 wait complete. 2023-11-17T05:23:30.514 INFO:teuthology.orchestra.run.smithi066.stderr:dumped monmap epoch 1 2023-11-17T05:23:30.739 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:23:30 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:23:29.177867+0000 mgr.smithi040.ogcprl (mgr.14164) 70 : cluster [DBG] pgmap v63: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:23:31.739 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:23:31 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:23:30.502346+0000 mon.smithi040 (mon.0) 146 : audit [DBG] from='client.? 172.21.15.66:0/2029749841' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-11-17T05:23:34.633 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2023-11-17T05:23:34.633 DEBUG:teuthology.orchestra.run.smithi066:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.4 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ed6470e2-8508-11ee-8dbe-212e2dc638e7 -- ceph mon dump -f json 2023-11-17T05:23:34.635 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:23:32 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:23:31.178182+0000 mgr.smithi040.ogcprl (mgr.14164) 71 : cluster [DBG] pgmap v64: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:23:34.738 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:23:34 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:23:33.178495+0000 mgr.smithi040.ogcprl (mgr.14164) 72 : cluster [DBG] pgmap v65: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:23:35.832 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:35.831+0000 7f08b708a700 1 -- 172.21.15.66:0/2225836297 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f08b0060280 msgr2=0x7f08b00f90f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:24:00.999 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:35.831+0000 7f08b708a700 1 --2- 172.21.15.66:0/2225836297 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f08b0060280 0x7f08b00f90f0 secure :-1 s=READY pgs=139 cs=0 l=1 rev1=1 rx=0x7f08a4009fb0 tx=0x7f08a4009b30).stop 2023-11-17T05:24:00.999 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:35.831+0000 7f08b708a700 1 -- 172.21.15.66:0/2225836297 shutdown_connections 2023-11-17T05:24:00.999 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:35.831+0000 7f08b708a700 1 --2- 172.21.15.66:0/2225836297 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f08b0060280 0x7f08b00f90f0 unknown :-1 s=CLOSED pgs=139 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:24:00.999 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:35.831+0000 7f08b708a700 1 -- 172.21.15.66:0/2225836297 >> 172.21.15.66:0/2225836297 conn(0x7f08b00f0b70 msgr2=0x7f08b00f2f90 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:24:00.999 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:35.831+0000 7f08b708a700 1 -- 172.21.15.66:0/2225836297 shutdown_connections 2023-11-17T05:24:00.999 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:35.831+0000 7f08b708a700 1 -- 172.21.15.66:0/2225836297 wait complete. 2023-11-17T05:24:00.999 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:35.832+0000 7f08b708a700 1 Processor -- start 2023-11-17T05:24:00.999 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:35.832+0000 7f08b708a700 1 -- start start 2023-11-17T05:24:01.002 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:23:36 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:23:35.178834+0000 mgr.smithi040.ogcprl (mgr.14164) 73 : cluster [DBG] pgmap v66: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:24:01.002 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:23:36 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:23:36.060258+0000 mon.smithi040 (mon.0) 147 : audit [DBG] from='client.? 172.21.15.66:0/3848358237' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-11-17T05:24:01.002 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:23:38 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:23:37.179193+0000 mgr.smithi040.ogcprl (mgr.14164) 74 : cluster [DBG] pgmap v67: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:24:01.002 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:23:40 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:23:39.179571+0000 mgr.smithi040.ogcprl (mgr.14164) 75 : cluster [DBG] pgmap v68: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:24:01.003 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:23:42 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:23:41.179902+0000 mgr.smithi040.ogcprl (mgr.14164) 76 : cluster [DBG] pgmap v69: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:24:01.003 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:23:44 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:23:43.180198+0000 mgr.smithi040.ogcprl (mgr.14164) 77 : cluster [DBG] pgmap v70: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:24:01.003 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:23:46 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:23:45.180539+0000 mgr.smithi040.ogcprl (mgr.14164) 78 : cluster [DBG] pgmap v71: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:24:01.005 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:23:47 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:23:47.180857+0000 mgr.smithi040.ogcprl (mgr.14164) 79 : cluster [DBG] pgmap v72: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:24:01.080 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:23:50 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:23:49.181213+0000 mgr.smithi040.ogcprl (mgr.14164) 80 : cluster [DBG] pgmap v73: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:24:01.080 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:23:52 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:23:51.181542+0000 mgr.smithi040.ogcprl (mgr.14164) 81 : cluster [DBG] pgmap v74: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:24:01.081 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:23:54 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:23:53.181877+0000 mgr.smithi040.ogcprl (mgr.14164) 82 : cluster [DBG] pgmap v75: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:24:01.081 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:23:56 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:23:55.182142+0000 mgr.smithi040.ogcprl (mgr.14164) 83 : cluster [DBG] pgmap v76: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:24:01.081 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:23:58 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:23:57.182477+0000 mgr.smithi040.ogcprl (mgr.14164) 84 : cluster [DBG] pgmap v77: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:24:01.081 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:24:00 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:23:59.182724+0000 mgr.smithi040.ogcprl (mgr.14164) 85 : cluster [DBG] pgmap v78: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:24:01.082 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:35.832+0000 7f08b708a700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f08b0060280 0x7f08b01088f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:24:01.082 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:35.832+0000 7f08b708a700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f08b0108de0 con 0x7f08b0060280 2023-11-17T05:24:01.082 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:35.832+0000 7f08b4e26700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f08b0060280 0x7f08b01088f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:24:01.082 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:35.832+0000 7f08b4e26700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f08b0060280 0x7f08b01088f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.66:60046/0 (socket says 172.21.15.66:60046) 2023-11-17T05:24:01.138 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:35.832+0000 7f08b4e26700 1 -- 172.21.15.66:0/3848358237 learned_addr learned my addr 172.21.15.66:0/3848358237 (peer_addr_for_me v2:172.21.15.66:0/0) 2023-11-17T05:24:01.139 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:35.833+0000 7f08b4e26700 1 -- 172.21.15.66:0/3848358237 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f08a4005040 con 0x7f08b0060280 2023-11-17T05:24:01.139 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:35.833+0000 7f08b4e26700 1 --2- 172.21.15.66:0/3848358237 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f08b0060280 0x7f08b01088f0 secure :-1 s=READY pgs=140 cs=0 l=1 rev1=1 rx=0x7f08a4013510 tx=0x7f08a400b040).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:24:01.139 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:35.833+0000 7f08ad7fa700 1 -- 172.21.15.66:0/3848358237 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f08a4009320 con 0x7f08b0060280 2023-11-17T05:24:01.139 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:35.833+0000 7f08b708a700 1 -- 172.21.15.66:0/3848358237 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f08b0108fe0 con 0x7f08b0060280 2023-11-17T05:24:01.169 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:35.833+0000 7f08ad7fa700 1 -- 172.21.15.66:0/3848358237 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(25 keys) v1 ==== 1074+0+0 (secure 0 0 0) 0x7f08a4013f00 con 0x7f08b0060280 2023-11-17T05:24:01.169 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:35.833+0000 7f08ad7fa700 1 -- 172.21.15.66:0/3848358237 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f08a4014dc0 con 0x7f08b0060280 2023-11-17T05:24:01.169 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:35.833+0000 7f08b708a700 1 -- 172.21.15.66:0/3848358237 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f08b0109470 con 0x7f08b0060280 2023-11-17T05:24:01.169 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:35.834+0000 7f08ad7fa700 1 -- 172.21.15.66:0/3848358237 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 17) v1 ==== 36269+0+0 (secure 0 0 0) 0x7f08a4014420 con 0x7f08b0060280 2023-11-17T05:24:01.169 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:35.834+0000 7f08ad7fa700 1 --2- 172.21.15.66:0/3848358237 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f08a002ff60 0x7f08a0032420 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:24:01.169 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:35.834+0000 7f08b708a700 1 -- 172.21.15.66:0/3848358237 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f08b0045ad0 con 0x7f08b0060280 2023-11-17T05:24:01.169 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:35.834+0000 7f08ad7fa700 1 -- 172.21.15.66:0/3848358237 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(5..5 src has 1..5) v4 ==== 1270+0+0 (secure 0 0 0) 0x7f08a40410d0 con 0x7f08b0060280 2023-11-17T05:24:01.169 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:35.834+0000 7f08affff700 1 --2- 172.21.15.66:0/3848358237 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f08a002ff60 0x7f08a0032420 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:24:01.169 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:35.835+0000 7f08affff700 1 --2- 172.21.15.66:0/3848358237 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f08a002ff60 0x7f08a0032420 secure :-1 s=READY pgs=35 cs=0 l=1 rev1=1 rx=0x7f089c0096a0 tx=0x7f089c006b40).ready entity=mgr.14164 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:24:01.169 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:35.839+0000 7f08ad7fa700 1 -- 172.21.15.66:0/3848358237 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+152212 (secure 0 0 0) 0x7f08a40095f0 con 0x7f08b0060280 2023-11-17T05:24:01.169 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:36.059+0000 7f08b708a700 1 -- 172.21.15.66:0/3848358237 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "mon dump", "format": "json"} v 0) v1 -- 0x7f08b00fc040 con 0x7f08b0060280 2023-11-17T05:24:01.170 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:36.060+0000 7f08ad7fa700 1 -- 172.21.15.66:0/3848358237 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mon_command_ack([{"prefix": "mon dump", "format": "json"}]=0 dumped monmap epoch 1 v1) v1 ==== 95+0+690 (secure 0 0 0) 0x7f08a401cb10 con 0x7f08b0060280 2023-11-17T05:24:01.173 INFO:teuthology.orchestra.run.smithi066.stdout: 2023-11-17T05:24:01.174 INFO:teuthology.orchestra.run.smithi066.stdout:{"epoch":1,"fsid":"ed6470e2-8508-11ee-8dbe-212e2dc638e7","modified":"2023-11-17T05:20:13.934038Z","created":"2023-11-17T05:20:13.934038Z","min_mon_release":16,"min_mon_release_name":"pacific","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging"],"optional":[]},"mons":[{"rank":0,"name":"smithi040","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:3300","nonce":0},{"type":"v1","addr":"172.21.15.40:6789","nonce":0}]},"addr":"172.21.15.40:6789/0","public_addr":"172.21.15.40:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2023-11-17T05:24:01.175 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:36.061+0000 7f08b708a700 1 -- 172.21.15.66:0/3848358237 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f08a002ff60 msgr2=0x7f08a0032420 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:24:01.175 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:36.061+0000 7f08b708a700 1 --2- 172.21.15.66:0/3848358237 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f08a002ff60 0x7f08a0032420 secure :-1 s=READY pgs=35 cs=0 l=1 rev1=1 rx=0x7f089c0096a0 tx=0x7f089c006b40).stop 2023-11-17T05:24:01.175 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:36.061+0000 7f08b708a700 1 -- 172.21.15.66:0/3848358237 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f08b0060280 msgr2=0x7f08b01088f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:24:01.175 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:36.061+0000 7f08b708a700 1 --2- 172.21.15.66:0/3848358237 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f08b0060280 0x7f08b01088f0 secure :-1 s=READY pgs=140 cs=0 l=1 rev1=1 rx=0x7f08a4013510 tx=0x7f08a400b040).stop 2023-11-17T05:24:01.175 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:36.061+0000 7f08b708a700 1 -- 172.21.15.66:0/3848358237 shutdown_connections 2023-11-17T05:24:01.175 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:36.061+0000 7f08b708a700 1 --2- 172.21.15.66:0/3848358237 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f08a002ff60 0x7f08a0032420 unknown :-1 s=CLOSED pgs=35 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:24:01.175 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:36.061+0000 7f08b708a700 1 --2- 172.21.15.66:0/3848358237 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f08b0060280 0x7f08b01088f0 unknown :-1 s=CLOSED pgs=140 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:24:01.176 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:36.061+0000 7f08b708a700 1 -- 172.21.15.66:0/3848358237 >> 172.21.15.66:0/3848358237 conn(0x7f08b00f0b70 msgr2=0x7f08b00f1620 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:24:01.176 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:36.062+0000 7f08b708a700 1 -- 172.21.15.66:0/3848358237 shutdown_connections 2023-11-17T05:24:01.176 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:23:36.062+0000 7f08b708a700 1 -- 172.21.15.66:0/3848358237 wait complete. 2023-11-17T05:24:01.176 INFO:teuthology.orchestra.run.smithi066.stderr:dumped monmap epoch 1 2023-11-17T05:24:02.177 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2023-11-17T05:24:02.177 DEBUG:teuthology.orchestra.run.smithi066:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.4 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ed6470e2-8508-11ee-8dbe-212e2dc638e7 -- ceph mon dump -f json 2023-11-17T05:24:02.739 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:24:02 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:24:01.183067+0000 mgr.smithi040.ogcprl (mgr.14164) 86 : cluster [DBG] pgmap v79: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:24:03.327 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:03.326+0000 7f5466dbb700 1 -- 172.21.15.66:0/2772600190 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f54600f6510 msgr2=0x7f54600f6930 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:24:03.327 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:03.326+0000 7f5466dbb700 1 --2- 172.21.15.66:0/2772600190 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f54600f6510 0x7f54600f6930 secure :-1 s=READY pgs=141 cs=0 l=1 rev1=1 rx=0x7f5454009fb0 tx=0x7f5454009b30).stop 2023-11-17T05:24:03.327 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:03.326+0000 7f5466dbb700 1 -- 172.21.15.66:0/2772600190 shutdown_connections 2023-11-17T05:24:03.327 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:03.326+0000 7f5466dbb700 1 --2- 172.21.15.66:0/2772600190 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f54600f6510 0x7f54600f6930 unknown :-1 s=CLOSED pgs=141 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:24:03.328 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:03.326+0000 7f5466dbb700 1 -- 172.21.15.66:0/2772600190 >> 172.21.15.66:0/2772600190 conn(0x7f54600f1c70 msgr2=0x7f54600f4090 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:24:03.328 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:03.326+0000 7f5466dbb700 1 -- 172.21.15.66:0/2772600190 shutdown_connections 2023-11-17T05:24:03.328 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:03.326+0000 7f5466dbb700 1 -- 172.21.15.66:0/2772600190 wait complete. 2023-11-17T05:24:03.328 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:03.327+0000 7f5466dbb700 1 Processor -- start 2023-11-17T05:24:03.328 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:03.327+0000 7f5466dbb700 1 -- start start 2023-11-17T05:24:03.328 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:03.327+0000 7f5466dbb700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f54600f6510 0x7f5460106730 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:24:03.328 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:03.327+0000 7f5466dbb700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f5460106c20 con 0x7f54600f6510 2023-11-17T05:24:03.328 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:03.327+0000 7f5464b57700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f54600f6510 0x7f5460106730 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:24:03.328 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:03.327+0000 7f5464b57700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f54600f6510 0x7f5460106730 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.66:56796/0 (socket says 172.21.15.66:56796) 2023-11-17T05:24:03.328 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:03.327+0000 7f5464b57700 1 -- 172.21.15.66:0/1751694858 learned_addr learned my addr 172.21.15.66:0/1751694858 (peer_addr_for_me v2:172.21.15.66:0/0) 2023-11-17T05:24:03.330 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:03.327+0000 7f5464b57700 1 -- 172.21.15.66:0/1751694858 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f5454005040 con 0x7f54600f6510 2023-11-17T05:24:03.330 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:03.328+0000 7f5464b57700 1 --2- 172.21.15.66:0/1751694858 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f54600f6510 0x7f5460106730 secure :-1 s=READY pgs=142 cs=0 l=1 rev1=1 rx=0x7f5454013510 tx=0x7f545400b040).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:24:03.330 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:03.328+0000 7f545d7fa700 1 -- 172.21.15.66:0/1751694858 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f5454009320 con 0x7f54600f6510 2023-11-17T05:24:03.330 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:03.328+0000 7f5466dbb700 1 -- 172.21.15.66:0/1751694858 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f5460106e20 con 0x7f54600f6510 2023-11-17T05:24:03.330 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:03.328+0000 7f545d7fa700 1 -- 172.21.15.66:0/1751694858 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(25 keys) v1 ==== 1074+0+0 (secure 0 0 0) 0x7f5454013f00 con 0x7f54600f6510 2023-11-17T05:24:03.330 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:03.328+0000 7f545d7fa700 1 -- 172.21.15.66:0/1751694858 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f5454014d90 con 0x7f54600f6510 2023-11-17T05:24:03.330 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:03.328+0000 7f5466dbb700 1 -- 172.21.15.66:0/1751694858 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f54601072b0 con 0x7f54600f6510 2023-11-17T05:24:03.330 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:03.328+0000 7f545d7fa700 1 -- 172.21.15.66:0/1751694858 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 17) v1 ==== 36269+0+0 (secure 0 0 0) 0x7f5454014420 con 0x7f54600f6510 2023-11-17T05:24:03.330 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:03.329+0000 7f5466dbb700 1 -- 172.21.15.66:0/1751694858 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f5460045ad0 con 0x7f54600f6510 2023-11-17T05:24:03.330 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:03.329+0000 7f545d7fa700 1 --2- 172.21.15.66:0/1751694858 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f545002ff60 0x7f5450032420 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:24:03.330 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:03.329+0000 7f545d7fa700 1 -- 172.21.15.66:0/1751694858 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(5..5 src has 1..5) v4 ==== 1270+0+0 (secure 0 0 0) 0x7f5454041110 con 0x7f54600f6510 2023-11-17T05:24:03.331 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:03.329+0000 7f545ffff700 1 --2- 172.21.15.66:0/1751694858 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f545002ff60 0x7f5450032420 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:24:03.331 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:03.329+0000 7f545ffff700 1 --2- 172.21.15.66:0/1751694858 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f545002ff60 0x7f5450032420 secure :-1 s=READY pgs=36 cs=0 l=1 rev1=1 rx=0x7f544c0096a0 tx=0x7f544c006b40).ready entity=mgr.14164 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:24:03.334 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:03.333+0000 7f545d7fa700 1 -- 172.21.15.66:0/1751694858 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+152212 (secure 0 0 0) 0x7f54540095f0 con 0x7f54600f6510 2023-11-17T05:24:03.552 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:03.552+0000 7f5466dbb700 1 -- 172.21.15.66:0/1751694858 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "mon dump", "format": "json"} v 0) v1 -- 0x7f54600fbf60 con 0x7f54600f6510 2023-11-17T05:24:03.553 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:03.553+0000 7f545d7fa700 1 -- 172.21.15.66:0/1751694858 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mon_command_ack([{"prefix": "mon dump", "format": "json"}]=0 dumped monmap epoch 1 v1) v1 ==== 95+0+690 (secure 0 0 0) 0x7f545401db70 con 0x7f54600f6510 2023-11-17T05:24:03.554 INFO:teuthology.orchestra.run.smithi066.stdout: 2023-11-17T05:24:03.554 INFO:teuthology.orchestra.run.smithi066.stdout:{"epoch":1,"fsid":"ed6470e2-8508-11ee-8dbe-212e2dc638e7","modified":"2023-11-17T05:20:13.934038Z","created":"2023-11-17T05:20:13.934038Z","min_mon_release":16,"min_mon_release_name":"pacific","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging"],"optional":[]},"mons":[{"rank":0,"name":"smithi040","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:3300","nonce":0},{"type":"v1","addr":"172.21.15.40:6789","nonce":0}]},"addr":"172.21.15.40:6789/0","public_addr":"172.21.15.40:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2023-11-17T05:24:03.555 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:03.554+0000 7f5466dbb700 1 -- 172.21.15.66:0/1751694858 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f545002ff60 msgr2=0x7f5450032420 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:24:03.555 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:03.554+0000 7f5466dbb700 1 --2- 172.21.15.66:0/1751694858 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f545002ff60 0x7f5450032420 secure :-1 s=READY pgs=36 cs=0 l=1 rev1=1 rx=0x7f544c0096a0 tx=0x7f544c006b40).stop 2023-11-17T05:24:03.555 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:03.554+0000 7f5466dbb700 1 -- 172.21.15.66:0/1751694858 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f54600f6510 msgr2=0x7f5460106730 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:24:03.555 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:03.554+0000 7f5466dbb700 1 --2- 172.21.15.66:0/1751694858 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f54600f6510 0x7f5460106730 secure :-1 s=READY pgs=142 cs=0 l=1 rev1=1 rx=0x7f5454013510 tx=0x7f545400b040).stop 2023-11-17T05:24:03.556 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:03.554+0000 7f5466dbb700 1 -- 172.21.15.66:0/1751694858 shutdown_connections 2023-11-17T05:24:03.556 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:03.554+0000 7f5466dbb700 1 --2- 172.21.15.66:0/1751694858 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f545002ff60 0x7f5450032420 unknown :-1 s=CLOSED pgs=36 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:24:03.556 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:03.554+0000 7f5466dbb700 1 --2- 172.21.15.66:0/1751694858 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f54600f6510 0x7f5460106730 unknown :-1 s=CLOSED pgs=142 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:24:03.556 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:03.554+0000 7f5466dbb700 1 -- 172.21.15.66:0/1751694858 >> 172.21.15.66:0/1751694858 conn(0x7f54600f1c70 msgr2=0x7f54600f2760 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:24:03.556 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:03.554+0000 7f5466dbb700 1 -- 172.21.15.66:0/1751694858 shutdown_connections 2023-11-17T05:24:03.556 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:03.554+0000 7f5466dbb700 1 -- 172.21.15.66:0/1751694858 wait complete. 2023-11-17T05:24:03.556 INFO:teuthology.orchestra.run.smithi066.stderr:dumped monmap epoch 1 2023-11-17T05:24:04.739 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:24:04 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:24:03.183401+0000 mgr.smithi040.ogcprl (mgr.14164) 87 : cluster [DBG] pgmap v80: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:24:04.739 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:24:04 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:24:03.553340+0000 mon.smithi040 (mon.0) 148 : audit [DBG] from='client.? 172.21.15.66:0/1751694858' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-11-17T05:24:04.918 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2023-11-17T05:24:04.919 DEBUG:teuthology.orchestra.run.smithi066:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.4 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ed6470e2-8508-11ee-8dbe-212e2dc638e7 -- ceph mon dump -f json 2023-11-17T05:24:06.154 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:06.153+0000 7f92d755b700 1 -- 172.21.15.66:0/1496843594 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f92d00f4800 msgr2=0x7f92d00f4c20 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:24:06.154 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:06.153+0000 7f92d755b700 1 --2- 172.21.15.66:0/1496843594 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f92d00f4800 0x7f92d00f4c20 secure :-1 s=READY pgs=143 cs=0 l=1 rev1=1 rx=0x7f92cc009fb0 tx=0x7f92cc009b30).stop 2023-11-17T05:24:06.154 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:06.153+0000 7f92d755b700 1 -- 172.21.15.66:0/1496843594 shutdown_connections 2023-11-17T05:24:06.154 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:06.153+0000 7f92d755b700 1 --2- 172.21.15.66:0/1496843594 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f92d00f4800 0x7f92d00f4c20 unknown :-1 s=CLOSED pgs=143 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:24:06.155 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:06.153+0000 7f92d755b700 1 -- 172.21.15.66:0/1496843594 >> 172.21.15.66:0/1496843594 conn(0x7f92d00f0260 msgr2=0x7f92d00f2680 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:24:06.155 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:06.153+0000 7f92d755b700 1 -- 172.21.15.66:0/1496843594 shutdown_connections 2023-11-17T05:24:06.155 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:06.153+0000 7f92d755b700 1 -- 172.21.15.66:0/1496843594 wait complete. 2023-11-17T05:24:06.155 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:06.154+0000 7f92d755b700 1 Processor -- start 2023-11-17T05:24:06.155 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:06.154+0000 7f92d755b700 1 -- start start 2023-11-17T05:24:06.155 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:06.154+0000 7f92d755b700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f92d00f4800 0x7f92d01066b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:24:06.155 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:06.154+0000 7f92d755b700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f92d0106ba0 con 0x7f92d00f4800 2023-11-17T05:24:06.156 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:06.154+0000 7f92d52f7700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f92d00f4800 0x7f92d01066b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:24:06.156 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:06.154+0000 7f92d52f7700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f92d00f4800 0x7f92d01066b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.66:56812/0 (socket says 172.21.15.66:56812) 2023-11-17T05:24:06.156 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:06.154+0000 7f92d52f7700 1 -- 172.21.15.66:0/3053666085 learned_addr learned my addr 172.21.15.66:0/3053666085 (peer_addr_for_me v2:172.21.15.66:0/0) 2023-11-17T05:24:06.156 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:06.155+0000 7f92d52f7700 1 -- 172.21.15.66:0/3053666085 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f92cc005040 con 0x7f92d00f4800 2023-11-17T05:24:06.156 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:06.155+0000 7f92d52f7700 1 --2- 172.21.15.66:0/3053666085 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f92d00f4800 0x7f92d01066b0 secure :-1 s=READY pgs=144 cs=0 l=1 rev1=1 rx=0x7f92cc013510 tx=0x7f92cc00b040).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:24:06.157 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:06.155+0000 7f92c5ffb700 1 -- 172.21.15.66:0/3053666085 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f92cc009320 con 0x7f92d00f4800 2023-11-17T05:24:06.157 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:06.155+0000 7f92c5ffb700 1 -- 172.21.15.66:0/3053666085 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(25 keys) v1 ==== 1074+0+0 (secure 0 0 0) 0x7f92cc013f00 con 0x7f92d00f4800 2023-11-17T05:24:06.157 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:06.155+0000 7f92d755b700 1 -- 172.21.15.66:0/3053666085 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f92d0106da0 con 0x7f92d00f4800 2023-11-17T05:24:06.157 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:06.155+0000 7f92c5ffb700 1 -- 172.21.15.66:0/3053666085 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f92cc014dc0 con 0x7f92d00f4800 2023-11-17T05:24:06.157 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:06.155+0000 7f92d755b700 1 -- 172.21.15.66:0/3053666085 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f92d0107230 con 0x7f92d00f4800 2023-11-17T05:24:06.157 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:06.156+0000 7f92c5ffb700 1 -- 172.21.15.66:0/3053666085 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 17) v1 ==== 36269+0+0 (secure 0 0 0) 0x7f92cc0095f0 con 0x7f92d00f4800 2023-11-17T05:24:06.157 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:06.156+0000 7f92d755b700 1 -- 172.21.15.66:0/3053666085 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f92d0045ad0 con 0x7f92d00f4800 2023-11-17T05:24:06.158 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:06.156+0000 7f92c5ffb700 1 --2- 172.21.15.66:0/3053666085 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f92bc02ff60 0x7f92bc032420 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:24:06.158 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:06.156+0000 7f92c5ffb700 1 -- 172.21.15.66:0/3053666085 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(5..5 src has 1..5) v4 ==== 1270+0+0 (secure 0 0 0) 0x7f92cc041040 con 0x7f92d00f4800 2023-11-17T05:24:06.158 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:06.157+0000 7f92d4af6700 1 --2- 172.21.15.66:0/3053666085 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f92bc02ff60 0x7f92bc032420 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:24:06.158 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:06.157+0000 7f92d4af6700 1 --2- 172.21.15.66:0/3053666085 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f92bc02ff60 0x7f92bc032420 secure :-1 s=READY pgs=37 cs=0 l=1 rev1=1 rx=0x7f92c00096a0 tx=0x7f92c0006b40).ready entity=mgr.14164 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:24:06.162 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:06.161+0000 7f92c5ffb700 1 -- 172.21.15.66:0/3053666085 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+152212 (secure 0 0 0) 0x7f92cc014420 con 0x7f92d00f4800 2023-11-17T05:24:06.379 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:06.379+0000 7f92d755b700 1 -- 172.21.15.66:0/3053666085 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "mon dump", "format": "json"} v 0) v1 -- 0x7f92d01080e0 con 0x7f92d00f4800 2023-11-17T05:24:06.379 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:06.379+0000 7f92c5ffb700 1 -- 172.21.15.66:0/3053666085 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mon_command_ack([{"prefix": "mon dump", "format": "json"}]=0 dumped monmap epoch 1 v1) v1 ==== 95+0+690 (secure 0 0 0) 0x7f92cc010230 con 0x7f92d00f4800 2023-11-17T05:24:06.380 INFO:teuthology.orchestra.run.smithi066.stdout: 2023-11-17T05:24:06.380 INFO:teuthology.orchestra.run.smithi066.stdout:{"epoch":1,"fsid":"ed6470e2-8508-11ee-8dbe-212e2dc638e7","modified":"2023-11-17T05:20:13.934038Z","created":"2023-11-17T05:20:13.934038Z","min_mon_release":16,"min_mon_release_name":"pacific","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging"],"optional":[]},"mons":[{"rank":0,"name":"smithi040","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:3300","nonce":0},{"type":"v1","addr":"172.21.15.40:6789","nonce":0}]},"addr":"172.21.15.40:6789/0","public_addr":"172.21.15.40:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2023-11-17T05:24:06.381 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:06.381+0000 7f92d755b700 1 -- 172.21.15.66:0/3053666085 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f92bc02ff60 msgr2=0x7f92bc032420 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:24:06.381 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:06.381+0000 7f92d755b700 1 --2- 172.21.15.66:0/3053666085 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f92bc02ff60 0x7f92bc032420 secure :-1 s=READY pgs=37 cs=0 l=1 rev1=1 rx=0x7f92c00096a0 tx=0x7f92c0006b40).stop 2023-11-17T05:24:06.381 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:06.381+0000 7f92d755b700 1 -- 172.21.15.66:0/3053666085 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f92d00f4800 msgr2=0x7f92d01066b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:24:06.382 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:06.381+0000 7f92d755b700 1 --2- 172.21.15.66:0/3053666085 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f92d00f4800 0x7f92d01066b0 secure :-1 s=READY pgs=144 cs=0 l=1 rev1=1 rx=0x7f92cc013510 tx=0x7f92cc00b040).stop 2023-11-17T05:24:06.382 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:06.381+0000 7f92d755b700 1 -- 172.21.15.66:0/3053666085 shutdown_connections 2023-11-17T05:24:06.382 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:06.381+0000 7f92d755b700 1 --2- 172.21.15.66:0/3053666085 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f92bc02ff60 0x7f92bc032420 unknown :-1 s=CLOSED pgs=37 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:24:06.382 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:06.381+0000 7f92d755b700 1 --2- 172.21.15.66:0/3053666085 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f92d00f4800 0x7f92d01066b0 unknown :-1 s=CLOSED pgs=144 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:24:06.382 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:06.381+0000 7f92d755b700 1 -- 172.21.15.66:0/3053666085 >> 172.21.15.66:0/3053666085 conn(0x7f92d00f0260 msgr2=0x7f92d00f0da0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:24:06.382 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:06.381+0000 7f92d755b700 1 -- 172.21.15.66:0/3053666085 shutdown_connections 2023-11-17T05:24:06.382 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:06.381+0000 7f92d755b700 1 -- 172.21.15.66:0/3053666085 wait complete. 2023-11-17T05:24:06.382 INFO:teuthology.orchestra.run.smithi066.stderr:dumped monmap epoch 1 2023-11-17T05:24:06.739 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:24:06 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:24:05.183752+0000 mgr.smithi040.ogcprl (mgr.14164) 88 : cluster [DBG] pgmap v81: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:24:06.739 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:24:06 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:24:06.379913+0000 mon.smithi040 (mon.0) 149 : audit [DBG] from='client.? 172.21.15.66:0/3053666085' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-11-17T05:24:07.778 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2023-11-17T05:24:07.778 DEBUG:teuthology.orchestra.run.smithi066:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.4 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ed6470e2-8508-11ee-8dbe-212e2dc638e7 -- ceph mon dump -f json 2023-11-17T05:24:08.739 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:24:08 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:24:07.184019+0000 mgr.smithi040.ogcprl (mgr.14164) 89 : cluster [DBG] pgmap v82: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:24:09.007 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:09.006+0000 7f9fcd8a1700 1 -- 172.21.15.66:0/1718646737 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f9fc80f5410 msgr2=0x7f9fc80f5830 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:24:09.007 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:09.006+0000 7f9fcd8a1700 1 --2- 172.21.15.66:0/1718646737 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f9fc80f5410 0x7f9fc80f5830 secure :-1 s=READY pgs=145 cs=0 l=1 rev1=1 rx=0x7f9fbc009fb0 tx=0x7f9fbc009b30).stop 2023-11-17T05:24:09.007 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:09.006+0000 7f9fcd8a1700 1 -- 172.21.15.66:0/1718646737 shutdown_connections 2023-11-17T05:24:09.008 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:09.006+0000 7f9fcd8a1700 1 --2- 172.21.15.66:0/1718646737 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f9fc80f5410 0x7f9fc80f5830 unknown :-1 s=CLOSED pgs=145 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:24:09.008 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:09.006+0000 7f9fcd8a1700 1 -- 172.21.15.66:0/1718646737 >> 172.21.15.66:0/1718646737 conn(0x7f9fc80f0b70 msgr2=0x7f9fc80f2f90 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:24:09.008 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:09.006+0000 7f9fcd8a1700 1 -- 172.21.15.66:0/1718646737 shutdown_connections 2023-11-17T05:24:09.008 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:09.006+0000 7f9fcd8a1700 1 -- 172.21.15.66:0/1718646737 wait complete. 2023-11-17T05:24:09.008 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:09.007+0000 7f9fcd8a1700 1 Processor -- start 2023-11-17T05:24:09.008 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:09.007+0000 7f9fcd8a1700 1 -- start start 2023-11-17T05:24:09.008 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:09.007+0000 7f9fcd8a1700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f9fc80f5410 0x7f9fc8106730 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:24:09.008 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:09.007+0000 7f9fcd8a1700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f9fc8106c20 con 0x7f9fc80f5410 2023-11-17T05:24:09.010 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:09.007+0000 7f9fc6ffd700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f9fc80f5410 0x7f9fc8106730 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:24:09.010 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:09.007+0000 7f9fc6ffd700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f9fc80f5410 0x7f9fc8106730 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.66:56816/0 (socket says 172.21.15.66:56816) 2023-11-17T05:24:09.010 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:09.007+0000 7f9fc6ffd700 1 -- 172.21.15.66:0/4067923119 learned_addr learned my addr 172.21.15.66:0/4067923119 (peer_addr_for_me v2:172.21.15.66:0/0) 2023-11-17T05:24:09.010 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:09.008+0000 7f9fc6ffd700 1 -- 172.21.15.66:0/4067923119 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f9fbc005040 con 0x7f9fc80f5410 2023-11-17T05:24:09.010 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:09.008+0000 7f9fc6ffd700 1 --2- 172.21.15.66:0/4067923119 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f9fc80f5410 0x7f9fc8106730 secure :-1 s=READY pgs=146 cs=0 l=1 rev1=1 rx=0x7f9fbc014000 tx=0x7f9fbc00b040).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:24:09.010 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:09.008+0000 7f9faf7fe700 1 -- 172.21.15.66:0/4067923119 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f9fbc0095f0 con 0x7f9fc80f5410 2023-11-17T05:24:09.011 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:09.008+0000 7f9faf7fe700 1 -- 172.21.15.66:0/4067923119 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(25 keys) v1 ==== 1074+0+0 (secure 0 0 0) 0x7f9fbc007e70 con 0x7f9fc80f5410 2023-11-17T05:24:09.011 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:09.008+0000 7f9fcd8a1700 1 -- 172.21.15.66:0/4067923119 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f9fc8106e20 con 0x7f9fc80f5410 2023-11-17T05:24:09.011 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:09.008+0000 7f9faf7fe700 1 -- 172.21.15.66:0/4067923119 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f9fbc014ea0 con 0x7f9fc80f5410 2023-11-17T05:24:09.011 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:09.008+0000 7f9fcd8a1700 1 -- 172.21.15.66:0/4067923119 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f9fc81072b0 con 0x7f9fc80f5410 2023-11-17T05:24:09.011 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:09.009+0000 7f9faf7fe700 1 -- 172.21.15.66:0/4067923119 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 17) v1 ==== 36269+0+0 (secure 0 0 0) 0x7f9fbc025070 con 0x7f9fc80f5410 2023-11-17T05:24:09.011 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:09.009+0000 7f9fcd8a1700 1 -- 172.21.15.66:0/4067923119 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f9fc8045ad0 con 0x7f9fc80f5410 2023-11-17T05:24:09.011 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:09.009+0000 7f9faf7fe700 1 --2- 172.21.15.66:0/4067923119 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f9fb002ff60 0x7f9fb0032420 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:24:09.011 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:09.009+0000 7f9faf7fe700 1 -- 172.21.15.66:0/4067923119 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(5..5 src has 1..5) v4 ==== 1270+0+0 (secure 0 0 0) 0x7f9fbc0409e0 con 0x7f9fc80f5410 2023-11-17T05:24:09.011 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:09.009+0000 7f9fc67fc700 1 --2- 172.21.15.66:0/4067923119 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f9fb002ff60 0x7f9fb0032420 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:24:09.011 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:09.010+0000 7f9fc67fc700 1 --2- 172.21.15.66:0/4067923119 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f9fb002ff60 0x7f9fb0032420 secure :-1 s=READY pgs=38 cs=0 l=1 rev1=1 rx=0x7f9fb80096a0 tx=0x7f9fb8006b40).ready entity=mgr.14164 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:24:09.014 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:09.014+0000 7f9faf7fe700 1 -- 172.21.15.66:0/4067923119 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+152212 (secure 0 0 0) 0x7f9fbc00f4e0 con 0x7f9fc80f5410 2023-11-17T05:24:09.231 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:09.231+0000 7f9fcd8a1700 1 -- 172.21.15.66:0/4067923119 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "mon dump", "format": "json"} v 0) v1 -- 0x7f9fc80fae60 con 0x7f9fc80f5410 2023-11-17T05:24:09.232 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:09.232+0000 7f9faf7fe700 1 -- 172.21.15.66:0/4067923119 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mon_command_ack([{"prefix": "mon dump", "format": "json"}]=0 dumped monmap epoch 1 v1) v1 ==== 95+0+690 (secure 0 0 0) 0x7f9fbc00f300 con 0x7f9fc80f5410 2023-11-17T05:24:09.232 INFO:teuthology.orchestra.run.smithi066.stdout: 2023-11-17T05:24:09.232 INFO:teuthology.orchestra.run.smithi066.stdout:{"epoch":1,"fsid":"ed6470e2-8508-11ee-8dbe-212e2dc638e7","modified":"2023-11-17T05:20:13.934038Z","created":"2023-11-17T05:20:13.934038Z","min_mon_release":16,"min_mon_release_name":"pacific","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging"],"optional":[]},"mons":[{"rank":0,"name":"smithi040","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:3300","nonce":0},{"type":"v1","addr":"172.21.15.40:6789","nonce":0}]},"addr":"172.21.15.40:6789/0","public_addr":"172.21.15.40:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2023-11-17T05:24:09.234 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:09.233+0000 7f9fcd8a1700 1 -- 172.21.15.66:0/4067923119 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f9fb002ff60 msgr2=0x7f9fb0032420 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:24:09.234 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:09.233+0000 7f9fcd8a1700 1 --2- 172.21.15.66:0/4067923119 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f9fb002ff60 0x7f9fb0032420 secure :-1 s=READY pgs=38 cs=0 l=1 rev1=1 rx=0x7f9fb80096a0 tx=0x7f9fb8006b40).stop 2023-11-17T05:24:09.234 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:09.233+0000 7f9fcd8a1700 1 -- 172.21.15.66:0/4067923119 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f9fc80f5410 msgr2=0x7f9fc8106730 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:24:09.234 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:09.233+0000 7f9fcd8a1700 1 --2- 172.21.15.66:0/4067923119 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f9fc80f5410 0x7f9fc8106730 secure :-1 s=READY pgs=146 cs=0 l=1 rev1=1 rx=0x7f9fbc014000 tx=0x7f9fbc00b040).stop 2023-11-17T05:24:09.234 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:09.233+0000 7f9fcd8a1700 1 -- 172.21.15.66:0/4067923119 shutdown_connections 2023-11-17T05:24:09.234 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:09.233+0000 7f9fcd8a1700 1 --2- 172.21.15.66:0/4067923119 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f9fb002ff60 0x7f9fb0032420 unknown :-1 s=CLOSED pgs=38 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:24:09.234 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:09.233+0000 7f9fcd8a1700 1 --2- 172.21.15.66:0/4067923119 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f9fc80f5410 0x7f9fc8106730 unknown :-1 s=CLOSED pgs=146 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:24:09.235 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:09.233+0000 7f9fcd8a1700 1 -- 172.21.15.66:0/4067923119 >> 172.21.15.66:0/4067923119 conn(0x7f9fc80f0b70 msgr2=0x7f9fc80f1660 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:24:09.235 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:09.233+0000 7f9fcd8a1700 1 -- 172.21.15.66:0/4067923119 shutdown_connections 2023-11-17T05:24:09.235 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:09.233+0000 7f9fcd8a1700 1 -- 172.21.15.66:0/4067923119 wait complete. 2023-11-17T05:24:09.235 INFO:teuthology.orchestra.run.smithi066.stderr:dumped monmap epoch 1 2023-11-17T05:24:09.739 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:24:09 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:24:09.232473+0000 mon.smithi040 (mon.0) 150 : audit [DBG] from='client.? 172.21.15.66:0/4067923119' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-11-17T05:24:10.628 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2023-11-17T05:24:10.628 DEBUG:teuthology.orchestra.run.smithi066:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.4 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ed6470e2-8508-11ee-8dbe-212e2dc638e7 -- ceph mon dump -f json 2023-11-17T05:24:10.740 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:24:10 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:24:09.184248+0000 mgr.smithi040.ogcprl (mgr.14164) 90 : cluster [DBG] pgmap v83: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:24:11.743 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:24:11 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:24:11.309609+0000 mon.smithi040 (mon.0) 151 : audit [INF] from='mgr.14164 172.21.15.40:0/2203032451' entity='mgr.smithi040.ogcprl' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi040.ogcprl/mirror_snapshot_schedule"}]: dispatch 2023-11-17T05:24:11.744 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:24:11 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:24:11.311981+0000 mon.smithi040 (mon.0) 152 : audit [INF] from='mgr.14164 172.21.15.40:0/2203032451' entity='mgr.smithi040.ogcprl' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi040.ogcprl/trash_purge_schedule"}]: dispatch 2023-11-17T05:24:11.853 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:11.850+0000 7f7ea2858700 1 -- 172.21.15.66:0/412254669 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f7e9c0f6510 msgr2=0x7f7e9c0f6930 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:24:11.853 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:11.850+0000 7f7ea2858700 1 --2- 172.21.15.66:0/412254669 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f7e9c0f6510 0x7f7e9c0f6930 secure :-1 s=READY pgs=147 cs=0 l=1 rev1=1 rx=0x7f7e8c009fb0 tx=0x7f7e8c009b30).stop 2023-11-17T05:24:11.853 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:11.850+0000 7f7ea2858700 1 -- 172.21.15.66:0/412254669 shutdown_connections 2023-11-17T05:24:11.853 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:11.850+0000 7f7ea2858700 1 --2- 172.21.15.66:0/412254669 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f7e9c0f6510 0x7f7e9c0f6930 unknown :-1 s=CLOSED pgs=147 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:24:11.853 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:11.850+0000 7f7ea2858700 1 -- 172.21.15.66:0/412254669 >> 172.21.15.66:0/412254669 conn(0x7f7e9c0f1c70 msgr2=0x7f7e9c0f4090 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:24:11.853 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:11.850+0000 7f7ea2858700 1 -- 172.21.15.66:0/412254669 shutdown_connections 2023-11-17T05:24:11.854 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:11.850+0000 7f7ea2858700 1 -- 172.21.15.66:0/412254669 wait complete. 2023-11-17T05:24:11.854 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:11.850+0000 7f7ea2858700 1 Processor -- start 2023-11-17T05:24:11.854 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:11.850+0000 7f7ea2858700 1 -- start start 2023-11-17T05:24:11.854 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:11.850+0000 7f7ea2858700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f7e9c0f6510 0x7f7e9c1066b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:24:11.855 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:11.850+0000 7f7ea2858700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f7e9c106ba0 con 0x7f7e9c0f6510 2023-11-17T05:24:11.855 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:11.851+0000 7f7e9bfff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f7e9c0f6510 0x7f7e9c1066b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:24:11.855 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:11.851+0000 7f7e9bfff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f7e9c0f6510 0x7f7e9c1066b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.66:56822/0 (socket says 172.21.15.66:56822) 2023-11-17T05:24:11.855 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:11.851+0000 7f7e9bfff700 1 -- 172.21.15.66:0/1566236739 learned_addr learned my addr 172.21.15.66:0/1566236739 (peer_addr_for_me v2:172.21.15.66:0/0) 2023-11-17T05:24:11.855 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:11.851+0000 7f7e9bfff700 1 -- 172.21.15.66:0/1566236739 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f7e8c005040 con 0x7f7e9c0f6510 2023-11-17T05:24:11.855 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:11.851+0000 7f7e9bfff700 1 --2- 172.21.15.66:0/1566236739 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f7e9c0f6510 0x7f7e9c1066b0 secure :-1 s=READY pgs=148 cs=0 l=1 rev1=1 rx=0x7f7e8c013510 tx=0x7f7e8c00b040).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:24:11.856 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:11.851+0000 7f7e98ff9700 1 -- 172.21.15.66:0/1566236739 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f7e8c009320 con 0x7f7e9c0f6510 2023-11-17T05:24:11.856 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:11.851+0000 7f7ea2858700 1 -- 172.21.15.66:0/1566236739 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f7e9c106da0 con 0x7f7e9c0f6510 2023-11-17T05:24:11.856 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:11.851+0000 7f7e98ff9700 1 -- 172.21.15.66:0/1566236739 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(25 keys) v1 ==== 1074+0+0 (secure 0 0 0) 0x7f7e8c013f00 con 0x7f7e9c0f6510 2023-11-17T05:24:11.856 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:11.851+0000 7f7e98ff9700 1 -- 172.21.15.66:0/1566236739 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f7e8c014d90 con 0x7f7e9c0f6510 2023-11-17T05:24:11.856 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:11.851+0000 7f7ea2858700 1 -- 172.21.15.66:0/1566236739 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f7e9c107230 con 0x7f7e9c0f6510 2023-11-17T05:24:11.856 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:11.852+0000 7f7e98ff9700 1 -- 172.21.15.66:0/1566236739 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 17) v1 ==== 36269+0+0 (secure 0 0 0) 0x7f7e8c0263a0 con 0x7f7e9c0f6510 2023-11-17T05:24:11.856 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:11.852+0000 7f7e98ff9700 1 --2- 172.21.15.66:0/1566236739 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f7e8802ffb0 0x7f7e88032470 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:24:11.856 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:11.852+0000 7f7ea2858700 1 -- 172.21.15.66:0/1566236739 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f7e9c045ad0 con 0x7f7e9c0f6510 2023-11-17T05:24:11.856 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:11.852+0000 7f7e98ff9700 1 -- 172.21.15.66:0/1566236739 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(5..5 src has 1..5) v4 ==== 1270+0+0 (secure 0 0 0) 0x7f7e8c040fe0 con 0x7f7e9c0f6510 2023-11-17T05:24:11.856 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:11.853+0000 7f7e9b7fe700 1 --2- 172.21.15.66:0/1566236739 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f7e8802ffb0 0x7f7e88032470 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:24:11.857 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:11.853+0000 7f7e9b7fe700 1 --2- 172.21.15.66:0/1566236739 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f7e8802ffb0 0x7f7e88032470 secure :-1 s=READY pgs=39 cs=0 l=1 rev1=1 rx=0x7f7e840096a0 tx=0x7f7e84006b40).ready entity=mgr.14164 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:24:11.858 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:11.858+0000 7f7e98ff9700 1 -- 172.21.15.66:0/1566236739 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+152212 (secure 0 0 0) 0x7f7e8c012920 con 0x7f7e9c0f6510 2023-11-17T05:24:12.076 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:12.075+0000 7f7ea2858700 1 -- 172.21.15.66:0/1566236739 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "mon dump", "format": "json"} v 0) v1 -- 0x7f7e9c1080e0 con 0x7f7e9c0f6510 2023-11-17T05:24:12.076 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:12.076+0000 7f7e98ff9700 1 -- 172.21.15.66:0/1566236739 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mon_command_ack([{"prefix": "mon dump", "format": "json"}]=0 dumped monmap epoch 1 v1) v1 ==== 95+0+690 (secure 0 0 0) 0x7f7e8c02b020 con 0x7f7e9c0f6510 2023-11-17T05:24:12.076 INFO:teuthology.orchestra.run.smithi066.stdout: 2023-11-17T05:24:12.077 INFO:teuthology.orchestra.run.smithi066.stdout:{"epoch":1,"fsid":"ed6470e2-8508-11ee-8dbe-212e2dc638e7","modified":"2023-11-17T05:20:13.934038Z","created":"2023-11-17T05:20:13.934038Z","min_mon_release":16,"min_mon_release_name":"pacific","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging"],"optional":[]},"mons":[{"rank":0,"name":"smithi040","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:3300","nonce":0},{"type":"v1","addr":"172.21.15.40:6789","nonce":0}]},"addr":"172.21.15.40:6789/0","public_addr":"172.21.15.40:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2023-11-17T05:24:12.078 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:12.078+0000 7f7ea2858700 1 -- 172.21.15.66:0/1566236739 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f7e8802ffb0 msgr2=0x7f7e88032470 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:24:12.078 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:12.078+0000 7f7ea2858700 1 --2- 172.21.15.66:0/1566236739 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f7e8802ffb0 0x7f7e88032470 secure :-1 s=READY pgs=39 cs=0 l=1 rev1=1 rx=0x7f7e840096a0 tx=0x7f7e84006b40).stop 2023-11-17T05:24:12.078 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:12.078+0000 7f7ea2858700 1 -- 172.21.15.66:0/1566236739 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f7e9c0f6510 msgr2=0x7f7e9c1066b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:24:12.078 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:12.078+0000 7f7ea2858700 1 --2- 172.21.15.66:0/1566236739 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f7e9c0f6510 0x7f7e9c1066b0 secure :-1 s=READY pgs=148 cs=0 l=1 rev1=1 rx=0x7f7e8c013510 tx=0x7f7e8c00b040).stop 2023-11-17T05:24:12.079 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:12.078+0000 7f7ea2858700 1 -- 172.21.15.66:0/1566236739 shutdown_connections 2023-11-17T05:24:12.079 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:12.078+0000 7f7ea2858700 1 --2- 172.21.15.66:0/1566236739 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f7e8802ffb0 0x7f7e88032470 unknown :-1 s=CLOSED pgs=39 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:24:12.079 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:12.078+0000 7f7ea2858700 1 --2- 172.21.15.66:0/1566236739 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f7e9c0f6510 0x7f7e9c1066b0 unknown :-1 s=CLOSED pgs=148 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:24:12.079 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:12.078+0000 7f7ea2858700 1 -- 172.21.15.66:0/1566236739 >> 172.21.15.66:0/1566236739 conn(0x7f7e9c0f1c70 msgr2=0x7f7e9c0f26d0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:24:12.079 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:12.078+0000 7f7ea2858700 1 -- 172.21.15.66:0/1566236739 shutdown_connections 2023-11-17T05:24:12.079 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:12.078+0000 7f7ea2858700 1 -- 172.21.15.66:0/1566236739 wait complete. 2023-11-17T05:24:12.079 INFO:teuthology.orchestra.run.smithi066.stderr:dumped monmap epoch 1 2023-11-17T05:24:12.739 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:24:12 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:24:11.184562+0000 mgr.smithi040.ogcprl (mgr.14164) 91 : cluster [DBG] pgmap v84: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:24:12.739 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:24:12 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:24:12.076616+0000 mon.smithi040 (mon.0) 153 : audit [DBG] from='client.? 172.21.15.66:0/1566236739' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-11-17T05:24:13.466 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2023-11-17T05:24:13.467 DEBUG:teuthology.orchestra.run.smithi066:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.4 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ed6470e2-8508-11ee-8dbe-212e2dc638e7 -- ceph mon dump -f json 2023-11-17T05:24:14.628 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:14.626+0000 7f32fe86d700 1 -- 172.21.15.66:0/1561856622 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f32f80f5410 msgr2=0x7f32f80f5830 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:24:14.628 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:14.626+0000 7f32fe86d700 1 --2- 172.21.15.66:0/1561856622 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f32f80f5410 0x7f32f80f5830 secure :-1 s=READY pgs=149 cs=0 l=1 rev1=1 rx=0x7f32e8009fb0 tx=0x7f32e8009b30).stop 2023-11-17T05:24:14.628 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:14.626+0000 7f32fe86d700 1 -- 172.21.15.66:0/1561856622 shutdown_connections 2023-11-17T05:24:14.629 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:14.626+0000 7f32fe86d700 1 --2- 172.21.15.66:0/1561856622 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f32f80f5410 0x7f32f80f5830 unknown :-1 s=CLOSED pgs=149 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:24:14.629 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:14.626+0000 7f32fe86d700 1 -- 172.21.15.66:0/1561856622 >> 172.21.15.66:0/1561856622 conn(0x7f32f80f0b70 msgr2=0x7f32f80f2f90 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:24:14.629 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:14.626+0000 7f32fe86d700 1 -- 172.21.15.66:0/1561856622 shutdown_connections 2023-11-17T05:24:14.629 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:14.626+0000 7f32fe86d700 1 -- 172.21.15.66:0/1561856622 wait complete. 2023-11-17T05:24:14.629 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:14.627+0000 7f32fe86d700 1 Processor -- start 2023-11-17T05:24:14.629 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:14.627+0000 7f32fe86d700 1 -- start start 2023-11-17T05:24:14.629 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:14.627+0000 7f32fe86d700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f32f80f5410 0x7f32f810b850 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:24:14.629 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:14.627+0000 7f32fe86d700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f32f8107240 con 0x7f32f80f5410 2023-11-17T05:24:14.629 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:14.628+0000 7f32f7fff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f32f80f5410 0x7f32f810b850 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:24:14.629 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:14.628+0000 7f32f7fff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f32f80f5410 0x7f32f810b850 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.66:49380/0 (socket says 172.21.15.66:49380) 2023-11-17T05:24:14.629 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:14.628+0000 7f32f7fff700 1 -- 172.21.15.66:0/3768138861 learned_addr learned my addr 172.21.15.66:0/3768138861 (peer_addr_for_me v2:172.21.15.66:0/0) 2023-11-17T05:24:14.630 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:14.628+0000 7f32f7fff700 1 -- 172.21.15.66:0/3768138861 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f32e8005040 con 0x7f32f80f5410 2023-11-17T05:24:14.630 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:14.628+0000 7f32f7fff700 1 --2- 172.21.15.66:0/3768138861 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f32f80f5410 0x7f32f810b850 secure :-1 s=READY pgs=150 cs=0 l=1 rev1=1 rx=0x7f32e8013510 tx=0x7f32e800b040).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:24:14.630 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:14.628+0000 7f32f4ff9700 1 -- 172.21.15.66:0/3768138861 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f32e8009320 con 0x7f32f80f5410 2023-11-17T05:24:14.631 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:14.628+0000 7f32f4ff9700 1 -- 172.21.15.66:0/3768138861 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(25 keys) v1 ==== 1074+0+0 (secure 0 0 0) 0x7f32e8013f00 con 0x7f32f80f5410 2023-11-17T05:24:14.631 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:14.628+0000 7f32fe86d700 1 -- 172.21.15.66:0/3768138861 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f32f8060220 con 0x7f32f80f5410 2023-11-17T05:24:14.631 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:14.628+0000 7f32f4ff9700 1 -- 172.21.15.66:0/3768138861 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f32e8014d90 con 0x7f32f80f5410 2023-11-17T05:24:14.631 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:14.628+0000 7f32fe86d700 1 -- 172.21.15.66:0/3768138861 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f32f810bd40 con 0x7f32f80f5410 2023-11-17T05:24:14.631 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:14.629+0000 7f32f4ff9700 1 -- 172.21.15.66:0/3768138861 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 17) v1 ==== 36269+0+0 (secure 0 0 0) 0x7f32e80263a0 con 0x7f32f80f5410 2023-11-17T05:24:14.631 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:14.629+0000 7f32f4ff9700 1 --2- 172.21.15.66:0/3768138861 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f32e002ff60 0x7f32e0032420 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:24:14.631 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:14.629+0000 7f32f4ff9700 1 -- 172.21.15.66:0/3768138861 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(5..5 src has 1..5) v4 ==== 1270+0+0 (secure 0 0 0) 0x7f32e8041000 con 0x7f32f80f5410 2023-11-17T05:24:14.631 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:14.629+0000 7f32fe86d700 1 -- 172.21.15.66:0/3768138861 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f32f8045ad0 con 0x7f32f80f5410 2023-11-17T05:24:14.631 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:14.629+0000 7f32f77fe700 1 --2- 172.21.15.66:0/3768138861 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f32e002ff60 0x7f32e0032420 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:24:14.631 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:14.630+0000 7f32f77fe700 1 --2- 172.21.15.66:0/3768138861 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f32e002ff60 0x7f32e0032420 secure :-1 s=READY pgs=40 cs=0 l=1 rev1=1 rx=0x7f32ec0096a0 tx=0x7f32ec006b40).ready entity=mgr.14164 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:24:14.634 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:14.634+0000 7f32f4ff9700 1 -- 172.21.15.66:0/3768138861 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+152212 (secure 0 0 0) 0x7f32e8012920 con 0x7f32f80f5410 2023-11-17T05:24:14.738 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:24:14 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:24:13.184851+0000 mgr.smithi040.ogcprl (mgr.14164) 92 : cluster [DBG] pgmap v85: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:24:14.857 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:14.857+0000 7f32fe86d700 1 -- 172.21.15.66:0/3768138861 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "mon dump", "format": "json"} v 0) v1 -- 0x7f32f810c470 con 0x7f32f80f5410 2023-11-17T05:24:14.858 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:14.857+0000 7f32f4ff9700 1 -- 172.21.15.66:0/3768138861 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mon_command_ack([{"prefix": "mon dump", "format": "json"}]=0 dumped monmap epoch 1 v1) v1 ==== 95+0+690 (secure 0 0 0) 0x7f32e802b020 con 0x7f32f80f5410 2023-11-17T05:24:14.858 INFO:teuthology.orchestra.run.smithi066.stdout: 2023-11-17T05:24:14.858 INFO:teuthology.orchestra.run.smithi066.stdout:{"epoch":1,"fsid":"ed6470e2-8508-11ee-8dbe-212e2dc638e7","modified":"2023-11-17T05:20:13.934038Z","created":"2023-11-17T05:20:13.934038Z","min_mon_release":16,"min_mon_release_name":"pacific","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging"],"optional":[]},"mons":[{"rank":0,"name":"smithi040","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:3300","nonce":0},{"type":"v1","addr":"172.21.15.40:6789","nonce":0}]},"addr":"172.21.15.40:6789/0","public_addr":"172.21.15.40:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2023-11-17T05:24:14.861 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:14.859+0000 7f32fe86d700 1 -- 172.21.15.66:0/3768138861 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f32e002ff60 msgr2=0x7f32e0032420 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:24:14.861 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:14.859+0000 7f32fe86d700 1 --2- 172.21.15.66:0/3768138861 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f32e002ff60 0x7f32e0032420 secure :-1 s=READY pgs=40 cs=0 l=1 rev1=1 rx=0x7f32ec0096a0 tx=0x7f32ec006b40).stop 2023-11-17T05:24:14.861 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:14.859+0000 7f32fe86d700 1 -- 172.21.15.66:0/3768138861 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f32f80f5410 msgr2=0x7f32f810b850 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:24:14.861 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:14.859+0000 7f32fe86d700 1 --2- 172.21.15.66:0/3768138861 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f32f80f5410 0x7f32f810b850 secure :-1 s=READY pgs=150 cs=0 l=1 rev1=1 rx=0x7f32e8013510 tx=0x7f32e800b040).stop 2023-11-17T05:24:14.861 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:14.859+0000 7f32fe86d700 1 -- 172.21.15.66:0/3768138861 shutdown_connections 2023-11-17T05:24:14.861 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:14.859+0000 7f32fe86d700 1 --2- 172.21.15.66:0/3768138861 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f32e002ff60 0x7f32e0032420 unknown :-1 s=CLOSED pgs=40 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:24:14.861 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:14.859+0000 7f32fe86d700 1 --2- 172.21.15.66:0/3768138861 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f32f80f5410 0x7f32f810b850 unknown :-1 s=CLOSED pgs=150 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:24:14.861 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:14.859+0000 7f32fe86d700 1 -- 172.21.15.66:0/3768138861 >> 172.21.15.66:0/3768138861 conn(0x7f32f80f0b70 msgr2=0x7f32f80f1660 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:24:14.861 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:14.859+0000 7f32fe86d700 1 -- 172.21.15.66:0/3768138861 shutdown_connections 2023-11-17T05:24:14.861 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:14.859+0000 7f32fe86d700 1 -- 172.21.15.66:0/3768138861 wait complete. 2023-11-17T05:24:14.861 INFO:teuthology.orchestra.run.smithi066.stderr:dumped monmap epoch 1 2023-11-17T05:24:15.739 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:24:15 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:24:14.858151+0000 mon.smithi040 (mon.0) 154 : audit [DBG] from='client.? 172.21.15.66:0/3768138861' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-11-17T05:24:16.237 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2023-11-17T05:24:16.237 DEBUG:teuthology.orchestra.run.smithi066:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.4 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ed6470e2-8508-11ee-8dbe-212e2dc638e7 -- ceph mon dump -f json 2023-11-17T05:24:16.739 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:24:16 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:24:15.185146+0000 mgr.smithi040.ogcprl (mgr.14164) 93 : cluster [DBG] pgmap v86: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:24:17.422 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:17.417+0000 7f477e540700 1 -- 172.21.15.66:0/424067268 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f47780f5410 msgr2=0x7f47780f5830 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:24:17.422 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:17.417+0000 7f477e540700 1 --2- 172.21.15.66:0/424067268 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f47780f5410 0x7f47780f5830 secure :-1 s=READY pgs=151 cs=0 l=1 rev1=1 rx=0x7f4768009fb0 tx=0x7f4768009b30).stop 2023-11-17T05:24:17.422 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:17.417+0000 7f477e540700 1 -- 172.21.15.66:0/424067268 shutdown_connections 2023-11-17T05:24:17.422 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:17.417+0000 7f477e540700 1 --2- 172.21.15.66:0/424067268 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f47780f5410 0x7f47780f5830 unknown :-1 s=CLOSED pgs=151 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:24:17.423 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:17.417+0000 7f477e540700 1 -- 172.21.15.66:0/424067268 >> 172.21.15.66:0/424067268 conn(0x7f47780f0b70 msgr2=0x7f47780f2f90 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:24:17.423 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:17.417+0000 7f477e540700 1 -- 172.21.15.66:0/424067268 shutdown_connections 2023-11-17T05:24:17.423 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:17.417+0000 7f477e540700 1 -- 172.21.15.66:0/424067268 wait complete. 2023-11-17T05:24:17.423 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:17.418+0000 7f477e540700 1 Processor -- start 2023-11-17T05:24:17.423 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:17.418+0000 7f477e540700 1 -- start start 2023-11-17T05:24:17.424 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:17.418+0000 7f477e540700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f47780f5410 0x7f4778106730 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:24:17.424 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:17.418+0000 7f477e540700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f4778106c20 con 0x7f47780f5410 2023-11-17T05:24:17.424 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:17.418+0000 7f4777fff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f47780f5410 0x7f4778106730 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:24:17.424 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:17.418+0000 7f4777fff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f47780f5410 0x7f4778106730 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.66:49400/0 (socket says 172.21.15.66:49400) 2023-11-17T05:24:17.424 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:17.418+0000 7f4777fff700 1 -- 172.21.15.66:0/1866291303 learned_addr learned my addr 172.21.15.66:0/1866291303 (peer_addr_for_me v2:172.21.15.66:0/0) 2023-11-17T05:24:17.425 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:17.419+0000 7f4777fff700 1 -- 172.21.15.66:0/1866291303 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f4768005040 con 0x7f47780f5410 2023-11-17T05:24:17.425 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:17.419+0000 7f4777fff700 1 --2- 172.21.15.66:0/1866291303 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f47780f5410 0x7f4778106730 secure :-1 s=READY pgs=152 cs=0 l=1 rev1=1 rx=0x7f4768013510 tx=0x7f476800b040).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:24:17.425 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:17.419+0000 7f4774ff9700 1 -- 172.21.15.66:0/1866291303 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f4768009320 con 0x7f47780f5410 2023-11-17T05:24:17.425 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:17.419+0000 7f4774ff9700 1 -- 172.21.15.66:0/1866291303 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(25 keys) v1 ==== 1074+0+0 (secure 0 0 0) 0x7f4768013f00 con 0x7f47780f5410 2023-11-17T05:24:17.425 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:17.419+0000 7f477e540700 1 -- 172.21.15.66:0/1866291303 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f4778106e20 con 0x7f47780f5410 2023-11-17T05:24:17.425 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:17.419+0000 7f4774ff9700 1 -- 172.21.15.66:0/1866291303 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f4768014d90 con 0x7f47780f5410 2023-11-17T05:24:17.426 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:17.419+0000 7f477e540700 1 -- 172.21.15.66:0/1866291303 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f47781072b0 con 0x7f47780f5410 2023-11-17T05:24:17.426 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:17.420+0000 7f4774ff9700 1 -- 172.21.15.66:0/1866291303 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 17) v1 ==== 36269+0+0 (secure 0 0 0) 0x7f47680095f0 con 0x7f47780f5410 2023-11-17T05:24:17.426 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:17.420+0000 7f477e540700 1 -- 172.21.15.66:0/1866291303 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f4778045ad0 con 0x7f47780f5410 2023-11-17T05:24:17.426 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:17.420+0000 7f4774ff9700 1 --2- 172.21.15.66:0/1866291303 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f476002ffb0 0x7f4760032470 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:24:17.426 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:17.420+0000 7f4774ff9700 1 -- 172.21.15.66:0/1866291303 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(5..5 src has 1..5) v4 ==== 1270+0+0 (secure 0 0 0) 0x7f4768041040 con 0x7f47780f5410 2023-11-17T05:24:17.427 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:17.420+0000 7f47777fe700 1 --2- 172.21.15.66:0/1866291303 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f476002ffb0 0x7f4760032470 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:24:17.427 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:17.421+0000 7f47777fe700 1 --2- 172.21.15.66:0/1866291303 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f476002ffb0 0x7f4760032470 secure :-1 s=READY pgs=41 cs=0 l=1 rev1=1 rx=0x7f476c0096a0 tx=0x7f476c006b40).ready entity=mgr.14164 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:24:17.427 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:17.425+0000 7f4774ff9700 1 -- 172.21.15.66:0/1866291303 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+152212 (secure 0 0 0) 0x7f4768014420 con 0x7f47780f5410 2023-11-17T05:24:17.641 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:17.641+0000 7f477e540700 1 -- 172.21.15.66:0/1866291303 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "mon dump", "format": "json"} v 0) v1 -- 0x7f47780fae60 con 0x7f47780f5410 2023-11-17T05:24:20.639 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:24:18 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:24:17.185478+0000 mgr.smithi040.ogcprl (mgr.14164) 94 : cluster [DBG] pgmap v87: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:24:20.639 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:24:18 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:24:17.642270+0000 mon.smithi040 (mon.0) 155 : audit [DBG] from='client.? 172.21.15.66:0/1866291303' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-11-17T05:24:20.639 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:17.642+0000 7f4774ff9700 1 -- 172.21.15.66:0/1866291303 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mon_command_ack([{"prefix": "mon dump", "format": "json"}]=0 dumped monmap epoch 1 v1) v1 ==== 95+0+690 (secure 0 0 0) 0x7f476801d7f0 con 0x7f47780f5410 2023-11-17T05:24:20.639 INFO:teuthology.orchestra.run.smithi066.stdout: 2023-11-17T05:24:20.639 INFO:teuthology.orchestra.run.smithi066.stdout:{"epoch":1,"fsid":"ed6470e2-8508-11ee-8dbe-212e2dc638e7","modified":"2023-11-17T05:20:13.934038Z","created":"2023-11-17T05:20:13.934038Z","min_mon_release":16,"min_mon_release_name":"pacific","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging"],"optional":[]},"mons":[{"rank":0,"name":"smithi040","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:3300","nonce":0},{"type":"v1","addr":"172.21.15.40:6789","nonce":0}]},"addr":"172.21.15.40:6789/0","public_addr":"172.21.15.40:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2023-11-17T05:24:20.640 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:17.643+0000 7f477e540700 1 -- 172.21.15.66:0/1866291303 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f476002ffb0 msgr2=0x7f4760032470 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:24:20.641 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:17.643+0000 7f477e540700 1 --2- 172.21.15.66:0/1866291303 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f476002ffb0 0x7f4760032470 secure :-1 s=READY pgs=41 cs=0 l=1 rev1=1 rx=0x7f476c0096a0 tx=0x7f476c006b40).stop 2023-11-17T05:24:20.641 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:17.643+0000 7f477e540700 1 -- 172.21.15.66:0/1866291303 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f47780f5410 msgr2=0x7f4778106730 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:24:20.641 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:17.643+0000 7f477e540700 1 --2- 172.21.15.66:0/1866291303 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f47780f5410 0x7f4778106730 secure :-1 s=READY pgs=152 cs=0 l=1 rev1=1 rx=0x7f4768013510 tx=0x7f476800b040).stop 2023-11-17T05:24:20.641 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:17.643+0000 7f477e540700 1 -- 172.21.15.66:0/1866291303 shutdown_connections 2023-11-17T05:24:20.641 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:17.643+0000 7f477e540700 1 --2- 172.21.15.66:0/1866291303 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f476002ffb0 0x7f4760032470 unknown :-1 s=CLOSED pgs=41 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:24:20.641 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:17.643+0000 7f477e540700 1 --2- 172.21.15.66:0/1866291303 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f47780f5410 0x7f4778106730 unknown :-1 s=CLOSED pgs=152 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:24:20.641 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:17.643+0000 7f477e540700 1 -- 172.21.15.66:0/1866291303 >> 172.21.15.66:0/1866291303 conn(0x7f47780f0b70 msgr2=0x7f47780f1660 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:24:20.641 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:17.643+0000 7f477e540700 1 -- 172.21.15.66:0/1866291303 shutdown_connections 2023-11-17T05:24:20.641 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:17.643+0000 7f477e540700 1 -- 172.21.15.66:0/1866291303 wait complete. 2023-11-17T05:24:20.641 INFO:teuthology.orchestra.run.smithi066.stderr:dumped monmap epoch 1 2023-11-17T05:24:20.739 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:24:20 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:24:19.185807+0000 mgr.smithi040.ogcprl (mgr.14164) 95 : cluster [DBG] pgmap v88: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:24:21.643 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2023-11-17T05:24:21.643 DEBUG:teuthology.orchestra.run.smithi066:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.4 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ed6470e2-8508-11ee-8dbe-212e2dc638e7 -- ceph mon dump -f json 2023-11-17T05:24:22.739 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:24:22 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:24:21.186124+0000 mgr.smithi040.ogcprl (mgr.14164) 96 : cluster [DBG] pgmap v89: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:24:22.867 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:22.865+0000 7f854c9f7700 1 -- 172.21.15.66:0/430232558 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f85480f62a0 msgr2=0x7f85480f66c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:24:25.057 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:22.865+0000 7f854c9f7700 1 --2- 172.21.15.66:0/430232558 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f85480f62a0 0x7f85480f66c0 secure :-1 s=READY pgs=153 cs=0 l=1 rev1=1 rx=0x7f8538009fb0 tx=0x7f8538009b30).stop 2023-11-17T05:24:25.057 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:22.865+0000 7f854c9f7700 1 -- 172.21.15.66:0/430232558 shutdown_connections 2023-11-17T05:24:25.057 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:22.865+0000 7f854c9f7700 1 --2- 172.21.15.66:0/430232558 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f85480f62a0 0x7f85480f66c0 unknown :-1 s=CLOSED pgs=153 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:24:25.058 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:22.865+0000 7f854c9f7700 1 -- 172.21.15.66:0/430232558 >> 172.21.15.66:0/430232558 conn(0x7f85480f1ac0 msgr2=0x7f85480f3ee0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:24:25.058 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:22.865+0000 7f854c9f7700 1 -- 172.21.15.66:0/430232558 shutdown_connections 2023-11-17T05:24:25.058 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:22.865+0000 7f854c9f7700 1 -- 172.21.15.66:0/430232558 wait complete. 2023-11-17T05:24:25.058 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:22.865+0000 7f854c9f7700 1 Processor -- start 2023-11-17T05:24:25.058 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:22.866+0000 7f854c9f7700 1 -- start start 2023-11-17T05:24:25.058 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:22.866+0000 7f854c9f7700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f85480f62a0 0x7f85481086b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:24:25.058 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:22.866+0000 7f854c9f7700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f8548108ba0 con 0x7f85480f62a0 2023-11-17T05:24:25.058 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:22.866+0000 7f8546d9d700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f85480f62a0 0x7f85481086b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:24:25.058 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:22.866+0000 7f8546d9d700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f85480f62a0 0x7f85481086b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.66:47178/0 (socket says 172.21.15.66:47178) 2023-11-17T05:24:25.058 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:22.866+0000 7f8546d9d700 1 -- 172.21.15.66:0/2303716076 learned_addr learned my addr 172.21.15.66:0/2303716076 (peer_addr_for_me v2:172.21.15.66:0/0) 2023-11-17T05:24:25.058 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:22.866+0000 7f8546d9d700 1 -- 172.21.15.66:0/2303716076 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f8538005040 con 0x7f85480f62a0 2023-11-17T05:24:25.059 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:22.866+0000 7f8546d9d700 1 --2- 172.21.15.66:0/2303716076 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f85480f62a0 0x7f85481086b0 secure :-1 s=READY pgs=154 cs=0 l=1 rev1=1 rx=0x7f8538013510 tx=0x7f853800b040).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:24:25.059 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:22.866+0000 7f853f7fe700 1 -- 172.21.15.66:0/2303716076 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f8538009320 con 0x7f85480f62a0 2023-11-17T05:24:25.059 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:22.867+0000 7f854c9f7700 1 -- 172.21.15.66:0/2303716076 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f8548108da0 con 0x7f85480f62a0 2023-11-17T05:24:25.059 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:22.867+0000 7f854c9f7700 1 -- 172.21.15.66:0/2303716076 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f8548109230 con 0x7f85480f62a0 2023-11-17T05:24:25.059 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:22.867+0000 7f853f7fe700 1 -- 172.21.15.66:0/2303716076 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(25 keys) v1 ==== 1074+0+0 (secure 0 0 0) 0x7f8538013f00 con 0x7f85480f62a0 2023-11-17T05:24:25.059 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:22.867+0000 7f853f7fe700 1 -- 172.21.15.66:0/2303716076 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f8538014cc0 con 0x7f85480f62a0 2023-11-17T05:24:25.060 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:24:23 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:24:23.091457+0000 mon.smithi040 (mon.0) 156 : audit [DBG] from='client.? 172.21.15.66:0/2303716076' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-11-17T05:24:25.060 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:24:24 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:24:23.186420+0000 mgr.smithi040.ogcprl (mgr.14164) 97 : cluster [DBG] pgmap v90: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:24:25.061 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:22.867+0000 7f854c9f7700 1 -- 172.21.15.66:0/2303716076 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f852c004fa0 con 0x7f85480f62a0 2023-11-17T05:24:25.061 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:22.868+0000 7f853f7fe700 1 -- 172.21.15.66:0/2303716076 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 17) v1 ==== 36269+0+0 (secure 0 0 0) 0x7f8538014420 con 0x7f85480f62a0 2023-11-17T05:24:25.061 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:22.868+0000 7f853f7fe700 1 --2- 172.21.15.66:0/2303716076 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f853402ff60 0x7f8534032420 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:24:25.061 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:22.868+0000 7f853f7fe700 1 -- 172.21.15.66:0/2303716076 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(5..5 src has 1..5) v4 ==== 1270+0+0 (secure 0 0 0) 0x7f8538040f50 con 0x7f85480f62a0 2023-11-17T05:24:25.061 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:22.868+0000 7f854659c700 1 --2- 172.21.15.66:0/2303716076 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f853402ff60 0x7f8534032420 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:24:25.062 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:22.868+0000 7f854659c700 1 --2- 172.21.15.66:0/2303716076 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f853402ff60 0x7f8534032420 secure :-1 s=READY pgs=42 cs=0 l=1 rev1=1 rx=0x7f85300096a0 tx=0x7f8530006b40).ready entity=mgr.14164 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:24:25.062 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:22.873+0000 7f853f7fe700 1 -- 172.21.15.66:0/2303716076 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+152212 (secure 0 0 0) 0x7f853806d2b0 con 0x7f85480f62a0 2023-11-17T05:24:25.062 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:23.090+0000 7f854c9f7700 1 -- 172.21.15.66:0/2303716076 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "mon dump", "format": "json"} v 0) v1 -- 0x7f852c005670 con 0x7f85480f62a0 2023-11-17T05:24:25.062 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:23.091+0000 7f853f7fe700 1 -- 172.21.15.66:0/2303716076 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mon_command_ack([{"prefix": "mon dump", "format": "json"}]=0 dumped monmap epoch 1 v1) v1 ==== 95+0+690 (secure 0 0 0) 0x7f85380095f0 con 0x7f85480f62a0 2023-11-17T05:24:25.062 INFO:teuthology.orchestra.run.smithi066.stdout: 2023-11-17T05:24:25.062 INFO:teuthology.orchestra.run.smithi066.stdout:{"epoch":1,"fsid":"ed6470e2-8508-11ee-8dbe-212e2dc638e7","modified":"2023-11-17T05:20:13.934038Z","created":"2023-11-17T05:20:13.934038Z","min_mon_release":16,"min_mon_release_name":"pacific","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging"],"optional":[]},"mons":[{"rank":0,"name":"smithi040","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:3300","nonce":0},{"type":"v1","addr":"172.21.15.40:6789","nonce":0}]},"addr":"172.21.15.40:6789/0","public_addr":"172.21.15.40:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2023-11-17T05:24:25.063 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:23.092+0000 7f854c9f7700 1 -- 172.21.15.66:0/2303716076 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f853402ff60 msgr2=0x7f8534032420 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:24:25.064 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:23.092+0000 7f854c9f7700 1 --2- 172.21.15.66:0/2303716076 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f853402ff60 0x7f8534032420 secure :-1 s=READY pgs=42 cs=0 l=1 rev1=1 rx=0x7f85300096a0 tx=0x7f8530006b40).stop 2023-11-17T05:24:25.064 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:23.092+0000 7f854c9f7700 1 -- 172.21.15.66:0/2303716076 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f85480f62a0 msgr2=0x7f85481086b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:24:25.064 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:23.092+0000 7f854c9f7700 1 --2- 172.21.15.66:0/2303716076 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f85480f62a0 0x7f85481086b0 secure :-1 s=READY pgs=154 cs=0 l=1 rev1=1 rx=0x7f8538013510 tx=0x7f853800b040).stop 2023-11-17T05:24:25.064 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:23.092+0000 7f854c9f7700 1 -- 172.21.15.66:0/2303716076 shutdown_connections 2023-11-17T05:24:25.064 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:23.092+0000 7f854c9f7700 1 --2- 172.21.15.66:0/2303716076 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f853402ff60 0x7f8534032420 unknown :-1 s=CLOSED pgs=42 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:24:25.064 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:23.092+0000 7f854c9f7700 1 --2- 172.21.15.66:0/2303716076 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f85480f62a0 0x7f85481086b0 unknown :-1 s=CLOSED pgs=154 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:24:25.064 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:23.092+0000 7f854c9f7700 1 -- 172.21.15.66:0/2303716076 >> 172.21.15.66:0/2303716076 conn(0x7f85480f1ac0 msgr2=0x7f85480f2520 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:24:25.064 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:23.092+0000 7f854c9f7700 1 -- 172.21.15.66:0/2303716076 shutdown_connections 2023-11-17T05:24:25.064 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:23.092+0000 7f854c9f7700 1 -- 172.21.15.66:0/2303716076 wait complete. 2023-11-17T05:24:25.064 INFO:teuthology.orchestra.run.smithi066.stderr:dumped monmap epoch 1 2023-11-17T05:24:26.070 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2023-11-17T05:24:26.070 DEBUG:teuthology.orchestra.run.smithi066:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.4 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ed6470e2-8508-11ee-8dbe-212e2dc638e7 -- ceph mon dump -f json 2023-11-17T05:24:26.741 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:24:26 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:24:25.186770+0000 mgr.smithi040.ogcprl (mgr.14164) 98 : cluster [DBG] pgmap v91: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:24:27.292 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:27.292+0000 7fae352a5700 1 -- 172.21.15.66:0/3100638413 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fae30060200 msgr2=0x7fae300f9090 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:24:27.292 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:27.292+0000 7fae352a5700 1 --2- 172.21.15.66:0/3100638413 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fae30060200 0x7fae300f9090 secure :-1 s=READY pgs=155 cs=0 l=1 rev1=1 rx=0x7fae20009fb0 tx=0x7fae20009b30).stop 2023-11-17T05:24:27.292 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:27.292+0000 7fae352a5700 1 -- 172.21.15.66:0/3100638413 shutdown_connections 2023-11-17T05:24:27.292 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:27.292+0000 7fae352a5700 1 --2- 172.21.15.66:0/3100638413 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fae30060200 0x7fae300f9090 unknown :-1 s=CLOSED pgs=155 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:24:27.293 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:27.292+0000 7fae352a5700 1 -- 172.21.15.66:0/3100638413 >> 172.21.15.66:0/3100638413 conn(0x7fae300f0a90 msgr2=0x7fae300f2eb0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:24:27.293 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:27.292+0000 7fae352a5700 1 -- 172.21.15.66:0/3100638413 shutdown_connections 2023-11-17T05:24:27.293 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:27.292+0000 7fae352a5700 1 -- 172.21.15.66:0/3100638413 wait complete. 2023-11-17T05:24:27.294 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:27.293+0000 7fae352a5700 1 Processor -- start 2023-11-17T05:24:27.294 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:27.293+0000 7fae352a5700 1 -- start start 2023-11-17T05:24:27.294 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:27.293+0000 7fae352a5700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fae30060200 0x7fae30106930 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:24:27.294 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:27.293+0000 7fae352a5700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fae30106e20 con 0x7fae30060200 2023-11-17T05:24:27.294 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:27.293+0000 7fae2effd700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fae30060200 0x7fae30106930 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:24:27.294 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:27.293+0000 7fae2effd700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fae30060200 0x7fae30106930 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.66:47188/0 (socket says 172.21.15.66:47188) 2023-11-17T05:24:27.294 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:27.293+0000 7fae2effd700 1 -- 172.21.15.66:0/1953696633 learned_addr learned my addr 172.21.15.66:0/1953696633 (peer_addr_for_me v2:172.21.15.66:0/0) 2023-11-17T05:24:27.294 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:27.293+0000 7fae2effd700 1 -- 172.21.15.66:0/1953696633 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fae20005040 con 0x7fae30060200 2023-11-17T05:24:27.295 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:27.293+0000 7fae2effd700 1 --2- 172.21.15.66:0/1953696633 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fae30060200 0x7fae30106930 secure :-1 s=READY pgs=156 cs=0 l=1 rev1=1 rx=0x7fae20013510 tx=0x7fae2000b040).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:24:27.295 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:27.294+0000 7fae177fe700 1 -- 172.21.15.66:0/1953696633 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7fae20009320 con 0x7fae30060200 2023-11-17T05:24:27.295 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:27.294+0000 7fae177fe700 1 -- 172.21.15.66:0/1953696633 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(25 keys) v1 ==== 1074+0+0 (secure 0 0 0) 0x7fae20013f00 con 0x7fae30060200 2023-11-17T05:24:27.295 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:27.294+0000 7fae352a5700 1 -- 172.21.15.66:0/1953696633 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fae30107020 con 0x7fae30060200 2023-11-17T05:24:27.295 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:27.294+0000 7fae177fe700 1 -- 172.21.15.66:0/1953696633 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7fae20014dc0 con 0x7fae30060200 2023-11-17T05:24:27.295 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:27.294+0000 7fae352a5700 1 -- 172.21.15.66:0/1953696633 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fae301074b0 con 0x7fae30060200 2023-11-17T05:24:27.296 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:27.294+0000 7fae177fe700 1 -- 172.21.15.66:0/1953696633 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 17) v1 ==== 36269+0+0 (secure 0 0 0) 0x7fae2001d3d0 con 0x7fae30060200 2023-11-17T05:24:27.296 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:27.295+0000 7fae352a5700 1 -- 172.21.15.66:0/1953696633 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fae30045ad0 con 0x7fae30060200 2023-11-17T05:24:27.296 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:27.295+0000 7fae177fe700 1 --2- 172.21.15.66:0/1953696633 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fae1802ff60 0x7fae18032420 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:24:27.296 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:27.295+0000 7fae177fe700 1 -- 172.21.15.66:0/1953696633 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(5..5 src has 1..5) v4 ==== 1270+0+0 (secure 0 0 0) 0x7fae20041060 con 0x7fae30060200 2023-11-17T05:24:27.296 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:27.295+0000 7fae2e7fc700 1 --2- 172.21.15.66:0/1953696633 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fae1802ff60 0x7fae18032420 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:24:27.296 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:27.295+0000 7fae2e7fc700 1 --2- 172.21.15.66:0/1953696633 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fae1802ff60 0x7fae18032420 secure :-1 s=READY pgs=43 cs=0 l=1 rev1=1 rx=0x7fae240096a0 tx=0x7fae24006b40).ready entity=mgr.14164 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:24:27.300 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:27.300+0000 7fae177fe700 1 -- 172.21.15.66:0/1953696633 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+152212 (secure 0 0 0) 0x7fae200126f0 con 0x7fae30060200 2023-11-17T05:24:27.519 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:27.518+0000 7fae352a5700 1 -- 172.21.15.66:0/1953696633 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "mon dump", "format": "json"} v 0) v1 -- 0x7fae3005ea50 con 0x7fae30060200 2023-11-17T05:24:27.519 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:27.519+0000 7fae177fe700 1 -- 172.21.15.66:0/1953696633 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mon_command_ack([{"prefix": "mon dump", "format": "json"}]=0 dumped monmap epoch 1 v1) v1 ==== 95+0+690 (secure 0 0 0) 0x7fae20012330 con 0x7fae30060200 2023-11-17T05:24:27.520 INFO:teuthology.orchestra.run.smithi066.stdout: 2023-11-17T05:24:27.520 INFO:teuthology.orchestra.run.smithi066.stdout:{"epoch":1,"fsid":"ed6470e2-8508-11ee-8dbe-212e2dc638e7","modified":"2023-11-17T05:20:13.934038Z","created":"2023-11-17T05:20:13.934038Z","min_mon_release":16,"min_mon_release_name":"pacific","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging"],"optional":[]},"mons":[{"rank":0,"name":"smithi040","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:3300","nonce":0},{"type":"v1","addr":"172.21.15.40:6789","nonce":0}]},"addr":"172.21.15.40:6789/0","public_addr":"172.21.15.40:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2023-11-17T05:24:27.521 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:27.521+0000 7fae352a5700 1 -- 172.21.15.66:0/1953696633 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fae1802ff60 msgr2=0x7fae18032420 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:24:27.522 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:27.521+0000 7fae352a5700 1 --2- 172.21.15.66:0/1953696633 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fae1802ff60 0x7fae18032420 secure :-1 s=READY pgs=43 cs=0 l=1 rev1=1 rx=0x7fae240096a0 tx=0x7fae24006b40).stop 2023-11-17T05:24:27.522 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:27.521+0000 7fae352a5700 1 -- 172.21.15.66:0/1953696633 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fae30060200 msgr2=0x7fae30106930 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:24:27.522 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:27.521+0000 7fae352a5700 1 --2- 172.21.15.66:0/1953696633 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fae30060200 0x7fae30106930 secure :-1 s=READY pgs=156 cs=0 l=1 rev1=1 rx=0x7fae20013510 tx=0x7fae2000b040).stop 2023-11-17T05:24:27.522 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:27.521+0000 7fae352a5700 1 -- 172.21.15.66:0/1953696633 shutdown_connections 2023-11-17T05:24:27.522 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:27.521+0000 7fae352a5700 1 --2- 172.21.15.66:0/1953696633 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fae1802ff60 0x7fae18032420 unknown :-1 s=CLOSED pgs=43 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:24:27.522 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:27.521+0000 7fae352a5700 1 --2- 172.21.15.66:0/1953696633 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fae30060200 0x7fae30106930 unknown :-1 s=CLOSED pgs=156 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:24:27.522 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:27.521+0000 7fae352a5700 1 -- 172.21.15.66:0/1953696633 >> 172.21.15.66:0/1953696633 conn(0x7fae300f0a90 msgr2=0x7fae300f1570 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:24:27.522 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:27.521+0000 7fae352a5700 1 -- 172.21.15.66:0/1953696633 shutdown_connections 2023-11-17T05:24:27.522 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:27.521+0000 7fae352a5700 1 -- 172.21.15.66:0/1953696633 wait complete. 2023-11-17T05:24:27.523 INFO:teuthology.orchestra.run.smithi066.stderr:dumped monmap epoch 1 2023-11-17T05:24:28.739 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:24:28 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:24:27.187110+0000 mgr.smithi040.ogcprl (mgr.14164) 99 : cluster [DBG] pgmap v92: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:24:28.751 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:24:28 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:24:27.519911+0000 mon.smithi040 (mon.0) 157 : audit [DBG] from='client.? 172.21.15.66:0/1953696633' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-11-17T05:24:28.891 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2023-11-17T05:24:28.892 DEBUG:teuthology.orchestra.run.smithi066:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.4 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ed6470e2-8508-11ee-8dbe-212e2dc638e7 -- ceph mon dump -f json 2023-11-17T05:24:30.167 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:30.166+0000 7f373937d700 1 -- 172.21.15.66:0/1683785162 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f37340f5410 msgr2=0x7f37340f5830 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:24:30.168 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:30.166+0000 7f373937d700 1 --2- 172.21.15.66:0/1683785162 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f37340f5410 0x7f37340f5830 secure :-1 s=READY pgs=157 cs=0 l=1 rev1=1 rx=0x7f3724009960 tx=0x7f3724004d10).stop 2023-11-17T05:24:30.168 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:30.166+0000 7f373937d700 1 -- 172.21.15.66:0/1683785162 shutdown_connections 2023-11-17T05:24:30.168 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:30.166+0000 7f373937d700 1 --2- 172.21.15.66:0/1683785162 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f37340f5410 0x7f37340f5830 unknown :-1 s=CLOSED pgs=157 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:24:30.168 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:30.166+0000 7f373937d700 1 -- 172.21.15.66:0/1683785162 >> 172.21.15.66:0/1683785162 conn(0x7f37340f0b70 msgr2=0x7f37340f2f90 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:24:30.168 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:30.166+0000 7f373937d700 1 -- 172.21.15.66:0/1683785162 shutdown_connections 2023-11-17T05:24:30.168 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:30.166+0000 7f373937d700 1 -- 172.21.15.66:0/1683785162 wait complete. 2023-11-17T05:24:30.168 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:30.167+0000 7f373937d700 1 Processor -- start 2023-11-17T05:24:30.168 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:30.167+0000 7f373937d700 1 -- start start 2023-11-17T05:24:30.168 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:30.167+0000 7f373937d700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f37340f5410 0x7f3734106760 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:24:30.168 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:30.167+0000 7f373937d700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f3734106c50 con 0x7f37340f5410 2023-11-17T05:24:30.168 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:30.167+0000 7f3732ffd700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f37340f5410 0x7f3734106760 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:24:30.169 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:30.167+0000 7f3732ffd700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f37340f5410 0x7f3734106760 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.66:47202/0 (socket says 172.21.15.66:47202) 2023-11-17T05:24:30.169 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:30.167+0000 7f3732ffd700 1 -- 172.21.15.66:0/3378872793 learned_addr learned my addr 172.21.15.66:0/3378872793 (peer_addr_for_me v2:172.21.15.66:0/0) 2023-11-17T05:24:30.170 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:30.168+0000 7f3732ffd700 1 -- 172.21.15.66:0/3378872793 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f3724009640 con 0x7f37340f5410 2023-11-17T05:24:30.170 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:30.168+0000 7f3732ffd700 1 --2- 172.21.15.66:0/3378872793 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f37340f5410 0x7f3734106760 secure :-1 s=READY pgs=158 cs=0 l=1 rev1=1 rx=0x7f3724009610 tx=0x7f3724004d10).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:24:30.170 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:30.168+0000 7f372b7fe700 1 -- 172.21.15.66:0/3378872793 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f3724018070 con 0x7f37340f5410 2023-11-17T05:24:30.170 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:30.168+0000 7f373937d700 1 -- 172.21.15.66:0/3378872793 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f3734106e50 con 0x7f37340f5410 2023-11-17T05:24:30.170 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:30.168+0000 7f372b7fe700 1 -- 172.21.15.66:0/3378872793 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(25 keys) v1 ==== 1074+0+0 (secure 0 0 0) 0x7f372400b870 con 0x7f37340f5410 2023-11-17T05:24:30.170 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:30.168+0000 7f372b7fe700 1 -- 172.21.15.66:0/3378872793 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f372401d420 con 0x7f37340f5410 2023-11-17T05:24:30.171 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:30.168+0000 7f373937d700 1 -- 172.21.15.66:0/3378872793 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f37341072e0 con 0x7f37340f5410 2023-11-17T05:24:30.171 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:30.169+0000 7f372b7fe700 1 -- 172.21.15.66:0/3378872793 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 17) v1 ==== 36269+0+0 (secure 0 0 0) 0x7f3724024060 con 0x7f37340f5410 2023-11-17T05:24:30.171 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:30.169+0000 7f373937d700 1 -- 172.21.15.66:0/3378872793 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f3734045ad0 con 0x7f37340f5410 2023-11-17T05:24:30.171 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:30.169+0000 7f372b7fe700 1 --2- 172.21.15.66:0/3378872793 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f372002ff60 0x7f3720032420 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:24:30.171 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:30.169+0000 7f372b7fe700 1 -- 172.21.15.66:0/3378872793 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(5..5 src has 1..5) v4 ==== 1270+0+0 (secure 0 0 0) 0x7f3724041170 con 0x7f37340f5410 2023-11-17T05:24:30.171 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:30.170+0000 7f37327fc700 1 --2- 172.21.15.66:0/3378872793 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f372002ff60 0x7f3720032420 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:24:30.171 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:30.170+0000 7f37327fc700 1 --2- 172.21.15.66:0/3378872793 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f372002ff60 0x7f3720032420 secure :-1 s=READY pgs=44 cs=0 l=1 rev1=1 rx=0x7f371c0096a0 tx=0x7f371c006b40).ready entity=mgr.14164 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:24:30.174 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:30.174+0000 7f372b7fe700 1 -- 172.21.15.66:0/3378872793 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+152212 (secure 0 0 0) 0x7f3724025730 con 0x7f37340f5410 2023-11-17T05:24:30.394 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:30.394+0000 7f373937d700 1 -- 172.21.15.66:0/3378872793 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "mon dump", "format": "json"} v 0) v1 -- 0x7f3734108190 con 0x7f37340f5410 2023-11-17T05:24:30.395 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:30.394+0000 7f372b7fe700 1 -- 172.21.15.66:0/3378872793 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mon_command_ack([{"prefix": "mon dump", "format": "json"}]=0 dumped monmap epoch 1 v1) v1 ==== 95+0+690 (secure 0 0 0) 0x7f3724025340 con 0x7f37340f5410 2023-11-17T05:24:30.395 INFO:teuthology.orchestra.run.smithi066.stdout: 2023-11-17T05:24:30.395 INFO:teuthology.orchestra.run.smithi066.stdout:{"epoch":1,"fsid":"ed6470e2-8508-11ee-8dbe-212e2dc638e7","modified":"2023-11-17T05:20:13.934038Z","created":"2023-11-17T05:20:13.934038Z","min_mon_release":16,"min_mon_release_name":"pacific","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging"],"optional":[]},"mons":[{"rank":0,"name":"smithi040","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:3300","nonce":0},{"type":"v1","addr":"172.21.15.40:6789","nonce":0}]},"addr":"172.21.15.40:6789/0","public_addr":"172.21.15.40:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2023-11-17T05:24:30.397 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:30.396+0000 7f373937d700 1 -- 172.21.15.66:0/3378872793 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f372002ff60 msgr2=0x7f3720032420 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:24:30.397 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:30.396+0000 7f373937d700 1 --2- 172.21.15.66:0/3378872793 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f372002ff60 0x7f3720032420 secure :-1 s=READY pgs=44 cs=0 l=1 rev1=1 rx=0x7f371c0096a0 tx=0x7f371c006b40).stop 2023-11-17T05:24:30.397 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:30.396+0000 7f373937d700 1 -- 172.21.15.66:0/3378872793 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f37340f5410 msgr2=0x7f3734106760 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:24:30.397 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:30.396+0000 7f373937d700 1 --2- 172.21.15.66:0/3378872793 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f37340f5410 0x7f3734106760 secure :-1 s=READY pgs=158 cs=0 l=1 rev1=1 rx=0x7f3724009610 tx=0x7f3724004d10).stop 2023-11-17T05:24:30.397 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:30.396+0000 7f373937d700 1 -- 172.21.15.66:0/3378872793 shutdown_connections 2023-11-17T05:24:30.397 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:30.396+0000 7f373937d700 1 --2- 172.21.15.66:0/3378872793 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f372002ff60 0x7f3720032420 unknown :-1 s=CLOSED pgs=44 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:24:30.397 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:30.396+0000 7f373937d700 1 --2- 172.21.15.66:0/3378872793 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f37340f5410 0x7f3734106760 unknown :-1 s=CLOSED pgs=158 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:24:30.397 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:30.396+0000 7f373937d700 1 -- 172.21.15.66:0/3378872793 >> 172.21.15.66:0/3378872793 conn(0x7f37340f0b70 msgr2=0x7f373405fe40 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:24:30.397 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:30.396+0000 7f373937d700 1 -- 172.21.15.66:0/3378872793 shutdown_connections 2023-11-17T05:24:30.397 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:30.396+0000 7f373937d700 1 -- 172.21.15.66:0/3378872793 wait complete. 2023-11-17T05:24:30.398 INFO:teuthology.orchestra.run.smithi066.stderr:dumped monmap epoch 1 2023-11-17T05:24:30.739 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:24:30 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:24:29.187496+0000 mgr.smithi040.ogcprl (mgr.14164) 100 : cluster [DBG] pgmap v93: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:24:30.739 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:24:30 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:24:30.395123+0000 mon.smithi040 (mon.0) 158 : audit [DBG] from='client.? 172.21.15.66:0/3378872793' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-11-17T05:24:31.767 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2023-11-17T05:24:31.767 DEBUG:teuthology.orchestra.run.smithi066:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.4 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ed6470e2-8508-11ee-8dbe-212e2dc638e7 -- ceph mon dump -f json 2023-11-17T05:24:32.739 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:24:32 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:24:31.187840+0000 mgr.smithi040.ogcprl (mgr.14164) 101 : cluster [DBG] pgmap v94: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:24:32.989 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:32.987+0000 7f6eb0e49700 1 -- 172.21.15.66:0/1101318240 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f6eac0f5410 msgr2=0x7f6eac0f5830 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:24:32.989 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:32.987+0000 7f6eb0e49700 1 --2- 172.21.15.66:0/1101318240 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f6eac0f5410 0x7f6eac0f5830 secure :-1 s=READY pgs=159 cs=0 l=1 rev1=1 rx=0x7f6e9c009fb0 tx=0x7f6e9c009b30).stop 2023-11-17T05:24:32.989 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:32.987+0000 7f6eb0e49700 1 -- 172.21.15.66:0/1101318240 shutdown_connections 2023-11-17T05:24:32.989 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:32.987+0000 7f6eb0e49700 1 --2- 172.21.15.66:0/1101318240 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f6eac0f5410 0x7f6eac0f5830 unknown :-1 s=CLOSED pgs=159 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:24:32.989 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:32.987+0000 7f6eb0e49700 1 -- 172.21.15.66:0/1101318240 >> 172.21.15.66:0/1101318240 conn(0x7f6eac0f0b70 msgr2=0x7f6eac0f2f90 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:24:32.989 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:32.987+0000 7f6eb0e49700 1 -- 172.21.15.66:0/1101318240 shutdown_connections 2023-11-17T05:24:32.989 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:32.988+0000 7f6eb0e49700 1 -- 172.21.15.66:0/1101318240 wait complete. 2023-11-17T05:24:32.989 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:32.988+0000 7f6eb0e49700 1 Processor -- start 2023-11-17T05:24:32.990 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:32.988+0000 7f6eb0e49700 1 -- start start 2023-11-17T05:24:32.990 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:32.988+0000 7f6eb0e49700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f6eac0f5410 0x7f6eac105630 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:24:32.990 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:32.989+0000 7f6eb0e49700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f6eac105b20 con 0x7f6eac0f5410 2023-11-17T05:24:32.990 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:32.989+0000 7f6eaa59c700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f6eac0f5410 0x7f6eac105630 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:24:32.990 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:32.989+0000 7f6eaa59c700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f6eac0f5410 0x7f6eac105630 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.66:33534/0 (socket says 172.21.15.66:33534) 2023-11-17T05:24:32.990 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:32.989+0000 7f6eaa59c700 1 -- 172.21.15.66:0/39543148 learned_addr learned my addr 172.21.15.66:0/39543148 (peer_addr_for_me v2:172.21.15.66:0/0) 2023-11-17T05:24:32.991 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:32.989+0000 7f6eaa59c700 1 -- 172.21.15.66:0/39543148 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f6e9c005040 con 0x7f6eac0f5410 2023-11-17T05:24:32.991 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:32.989+0000 7f6eaa59c700 1 --2- 172.21.15.66:0/39543148 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f6eac0f5410 0x7f6eac105630 secure :-1 s=READY pgs=160 cs=0 l=1 rev1=1 rx=0x7f6e9c013510 tx=0x7f6e9c00b040).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:24:32.991 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:32.989+0000 7f6e9affd700 1 -- 172.21.15.66:0/39543148 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f6e9c009320 con 0x7f6eac0f5410 2023-11-17T05:24:32.992 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:32.989+0000 7f6eb0e49700 1 -- 172.21.15.66:0/39543148 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f6eac105d20 con 0x7f6eac0f5410 2023-11-17T05:24:32.992 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:32.989+0000 7f6e9affd700 1 -- 172.21.15.66:0/39543148 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(25 keys) v1 ==== 1074+0+0 (secure 0 0 0) 0x7f6e9c013f00 con 0x7f6eac0f5410 2023-11-17T05:24:32.992 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:32.989+0000 7f6e9affd700 1 -- 172.21.15.66:0/39543148 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f6e9c014d90 con 0x7f6eac0f5410 2023-11-17T05:24:32.992 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:32.989+0000 7f6eb0e49700 1 -- 172.21.15.66:0/39543148 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f6eac1061b0 con 0x7f6eac0f5410 2023-11-17T05:24:32.992 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:32.990+0000 7f6e9affd700 1 -- 172.21.15.66:0/39543148 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 17) v1 ==== 36269+0+0 (secure 0 0 0) 0x7f6e9c0263a0 con 0x7f6eac0f5410 2023-11-17T05:24:32.992 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:32.990+0000 7f6eb0e49700 1 -- 172.21.15.66:0/39543148 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f6eac045ad0 con 0x7f6eac0f5410 2023-11-17T05:24:32.992 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:32.990+0000 7f6e9affd700 1 --2- 172.21.15.66:0/39543148 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f6e9402ff60 0x7f6e94032420 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:24:32.992 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:32.990+0000 7f6e9affd700 1 -- 172.21.15.66:0/39543148 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(5..5 src has 1..5) v4 ==== 1270+0+0 (secure 0 0 0) 0x7f6e9c0412f0 con 0x7f6eac0f5410 2023-11-17T05:24:32.992 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:32.991+0000 7f6ea9d9b700 1 --2- 172.21.15.66:0/39543148 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f6e9402ff60 0x7f6e94032420 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:24:32.992 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:32.991+0000 7f6ea9d9b700 1 --2- 172.21.15.66:0/39543148 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f6e9402ff60 0x7f6e94032420 secure :-1 s=READY pgs=45 cs=0 l=1 rev1=1 rx=0x7f6ea00096a0 tx=0x7f6ea0006b40).ready entity=mgr.14164 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:24:32.995 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:32.995+0000 7f6e9affd700 1 -- 172.21.15.66:0/39543148 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+152212 (secure 0 0 0) 0x7f6e9c012330 con 0x7f6eac0f5410 2023-11-17T05:24:33.214 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:33.211+0000 7f6eb0e49700 1 -- 172.21.15.66:0/39543148 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "mon dump", "format": "json"} v 0) v1 -- 0x7f6eac0fae60 con 0x7f6eac0f5410 2023-11-17T05:24:33.214 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:33.212+0000 7f6e9affd700 1 -- 172.21.15.66:0/39543148 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mon_command_ack([{"prefix": "mon dump", "format": "json"}]=0 dumped monmap epoch 1 v1) v1 ==== 95+0+690 (secure 0 0 0) 0x7f6e9c012330 con 0x7f6eac0f5410 2023-11-17T05:24:33.214 INFO:teuthology.orchestra.run.smithi066.stdout: 2023-11-17T05:24:33.214 INFO:teuthology.orchestra.run.smithi066.stdout:{"epoch":1,"fsid":"ed6470e2-8508-11ee-8dbe-212e2dc638e7","modified":"2023-11-17T05:20:13.934038Z","created":"2023-11-17T05:20:13.934038Z","min_mon_release":16,"min_mon_release_name":"pacific","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging"],"optional":[]},"mons":[{"rank":0,"name":"smithi040","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:3300","nonce":0},{"type":"v1","addr":"172.21.15.40:6789","nonce":0}]},"addr":"172.21.15.40:6789/0","public_addr":"172.21.15.40:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2023-11-17T05:24:33.215 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:33.213+0000 7f6eb0e49700 1 -- 172.21.15.66:0/39543148 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f6e9402ff60 msgr2=0x7f6e94032420 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:24:33.215 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:33.213+0000 7f6eb0e49700 1 --2- 172.21.15.66:0/39543148 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f6e9402ff60 0x7f6e94032420 secure :-1 s=READY pgs=45 cs=0 l=1 rev1=1 rx=0x7f6ea00096a0 tx=0x7f6ea0006b40).stop 2023-11-17T05:24:33.215 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:33.213+0000 7f6eb0e49700 1 -- 172.21.15.66:0/39543148 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f6eac0f5410 msgr2=0x7f6eac105630 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:24:33.215 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:33.213+0000 7f6eb0e49700 1 --2- 172.21.15.66:0/39543148 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f6eac0f5410 0x7f6eac105630 secure :-1 s=READY pgs=160 cs=0 l=1 rev1=1 rx=0x7f6e9c013510 tx=0x7f6e9c00b040).stop 2023-11-17T05:24:33.215 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:33.213+0000 7f6eb0e49700 1 -- 172.21.15.66:0/39543148 shutdown_connections 2023-11-17T05:24:33.216 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:33.213+0000 7f6eb0e49700 1 --2- 172.21.15.66:0/39543148 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f6e9402ff60 0x7f6e94032420 unknown :-1 s=CLOSED pgs=45 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:24:33.216 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:33.213+0000 7f6eb0e49700 1 --2- 172.21.15.66:0/39543148 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f6eac0f5410 0x7f6eac105630 unknown :-1 s=CLOSED pgs=160 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:24:33.216 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:33.213+0000 7f6eb0e49700 1 -- 172.21.15.66:0/39543148 >> 172.21.15.66:0/39543148 conn(0x7f6eac0f0b70 msgr2=0x7f6eac0f1660 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:24:33.216 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:33.213+0000 7f6eb0e49700 1 -- 172.21.15.66:0/39543148 shutdown_connections 2023-11-17T05:24:33.216 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:33.213+0000 7f6eb0e49700 1 -- 172.21.15.66:0/39543148 wait complete. 2023-11-17T05:24:33.216 INFO:teuthology.orchestra.run.smithi066.stderr:dumped monmap epoch 1 2023-11-17T05:24:33.739 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:24:33 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:24:33.212496+0000 mon.smithi040 (mon.0) 159 : audit [DBG] from='client.? 172.21.15.66:0/39543148' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-11-17T05:24:34.625 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2023-11-17T05:24:34.625 DEBUG:teuthology.orchestra.run.smithi066:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.4 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ed6470e2-8508-11ee-8dbe-212e2dc638e7 -- ceph mon dump -f json 2023-11-17T05:24:34.738 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:24:34 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:24:33.188177+0000 mgr.smithi040.ogcprl (mgr.14164) 102 : cluster [DBG] pgmap v95: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:24:35.820 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:35.817+0000 7f8b4c815700 1 -- 172.21.15.66:0/4150084704 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f8b48060180 msgr2=0x7f8b480f8f80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:24:35.820 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:35.817+0000 7f8b4c815700 1 --2- 172.21.15.66:0/4150084704 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f8b48060180 0x7f8b480f8f80 secure :-1 s=READY pgs=161 cs=0 l=1 rev1=1 rx=0x7f8b30009fb0 tx=0x7f8b30009b30).stop 2023-11-17T05:24:35.820 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:35.817+0000 7f8b4c815700 1 -- 172.21.15.66:0/4150084704 shutdown_connections 2023-11-17T05:24:35.820 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:35.817+0000 7f8b4c815700 1 --2- 172.21.15.66:0/4150084704 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f8b48060180 0x7f8b480f8f80 unknown :-1 s=CLOSED pgs=161 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:24:35.820 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:35.817+0000 7f8b4c815700 1 -- 172.21.15.66:0/4150084704 >> 172.21.15.66:0/4150084704 conn(0x7f8b480f09a0 msgr2=0x7f8b480f2dc0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:24:35.820 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:35.817+0000 7f8b4c815700 1 -- 172.21.15.66:0/4150084704 shutdown_connections 2023-11-17T05:24:35.820 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:35.818+0000 7f8b4c815700 1 -- 172.21.15.66:0/4150084704 wait complete. 2023-11-17T05:24:35.820 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:35.818+0000 7f8b4c815700 1 Processor -- start 2023-11-17T05:24:35.820 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:35.818+0000 7f8b4c815700 1 -- start start 2023-11-17T05:24:35.820 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:35.818+0000 7f8b4c815700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f8b480f69c0 0x7f8b480f4e80 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:24:35.820 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:35.818+0000 7f8b4c815700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f8b480f6de0 con 0x7f8b480f69c0 2023-11-17T05:24:35.821 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:35.819+0000 7f8b46d9d700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f8b480f69c0 0x7f8b480f4e80 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:24:35.821 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:35.819+0000 7f8b46d9d700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f8b480f69c0 0x7f8b480f4e80 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.66:33544/0 (socket says 172.21.15.66:33544) 2023-11-17T05:24:35.821 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:35.819+0000 7f8b46d9d700 1 -- 172.21.15.66:0/3512817534 learned_addr learned my addr 172.21.15.66:0/3512817534 (peer_addr_for_me v2:172.21.15.66:0/0) 2023-11-17T05:24:35.822 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:35.819+0000 7f8b46d9d700 1 -- 172.21.15.66:0/3512817534 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f8b30005040 con 0x7f8b480f69c0 2023-11-17T05:24:35.822 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:35.819+0000 7f8b46d9d700 1 --2- 172.21.15.66:0/3512817534 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f8b480f69c0 0x7f8b480f4e80 secure :-1 s=READY pgs=162 cs=0 l=1 rev1=1 rx=0x7f8b30009f80 tx=0x7f8b30009b30).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:24:35.822 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:35.819+0000 7f8b3f7fe700 1 -- 172.21.15.66:0/3512817534 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f8b300136c0 con 0x7f8b480f69c0 2023-11-17T05:24:35.822 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:35.819+0000 7f8b4c815700 1 -- 172.21.15.66:0/3512817534 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f8b480f5370 con 0x7f8b480f69c0 2023-11-17T05:24:35.822 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:35.819+0000 7f8b3f7fe700 1 -- 172.21.15.66:0/3512817534 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(25 keys) v1 ==== 1074+0+0 (secure 0 0 0) 0x7f8b300095f0 con 0x7f8b480f69c0 2023-11-17T05:24:35.822 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:35.819+0000 7f8b3f7fe700 1 -- 172.21.15.66:0/3512817534 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f8b30014420 con 0x7f8b480f69c0 2023-11-17T05:24:35.822 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:35.819+0000 7f8b4c815700 1 -- 172.21.15.66:0/3512817534 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f8b480f5860 con 0x7f8b480f69c0 2023-11-17T05:24:35.822 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:35.820+0000 7f8b3f7fe700 1 -- 172.21.15.66:0/3512817534 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 17) v1 ==== 36269+0+0 (secure 0 0 0) 0x7f8b30025060 con 0x7f8b480f69c0 2023-11-17T05:24:35.823 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:35.820+0000 7f8b3f7fe700 1 --2- 172.21.15.66:0/3512817534 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f8b3402ff60 0x7f8b34032420 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:24:35.823 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:35.820+0000 7f8b3f7fe700 1 -- 172.21.15.66:0/3512817534 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(5..5 src has 1..5) v4 ==== 1270+0+0 (secure 0 0 0) 0x7f8b30040e90 con 0x7f8b480f69c0 2023-11-17T05:24:35.823 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:35.820+0000 7f8b4c815700 1 -- 172.21.15.66:0/3512817534 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f8b48045ad0 con 0x7f8b480f69c0 2023-11-17T05:24:35.823 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:35.820+0000 7f8b4659c700 1 --2- 172.21.15.66:0/3512817534 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f8b3402ff60 0x7f8b34032420 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:24:35.823 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:35.821+0000 7f8b4659c700 1 --2- 172.21.15.66:0/3512817534 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f8b3402ff60 0x7f8b34032420 secure :-1 s=READY pgs=46 cs=0 l=1 rev1=1 rx=0x7f8b380096a0 tx=0x7f8b38006b40).ready entity=mgr.14164 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:24:35.826 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:35.825+0000 7f8b3f7fe700 1 -- 172.21.15.66:0/3512817534 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+152212 (secure 0 0 0) 0x7f8b30028440 con 0x7f8b480f69c0 2023-11-17T05:24:36.044 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:36.043+0000 7f8b4c815700 1 -- 172.21.15.66:0/3512817534 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "mon dump", "format": "json"} v 0) v1 -- 0x7f8b4810c810 con 0x7f8b480f69c0 2023-11-17T05:24:36.045 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:36.043+0000 7f8b3f7fe700 1 -- 172.21.15.66:0/3512817534 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mon_command_ack([{"prefix": "mon dump", "format": "json"}]=0 dumped monmap epoch 1 v1) v1 ==== 95+0+690 (secure 0 0 0) 0x7f8b3001d3a0 con 0x7f8b480f69c0 2023-11-17T05:24:36.045 INFO:teuthology.orchestra.run.smithi066.stdout: 2023-11-17T05:24:36.045 INFO:teuthology.orchestra.run.smithi066.stdout:{"epoch":1,"fsid":"ed6470e2-8508-11ee-8dbe-212e2dc638e7","modified":"2023-11-17T05:20:13.934038Z","created":"2023-11-17T05:20:13.934038Z","min_mon_release":16,"min_mon_release_name":"pacific","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging"],"optional":[]},"mons":[{"rank":0,"name":"smithi040","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:3300","nonce":0},{"type":"v1","addr":"172.21.15.40:6789","nonce":0}]},"addr":"172.21.15.40:6789/0","public_addr":"172.21.15.40:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2023-11-17T05:24:36.046 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:36.045+0000 7f8b4c815700 1 -- 172.21.15.66:0/3512817534 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f8b3402ff60 msgr2=0x7f8b34032420 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:24:36.046 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:36.045+0000 7f8b4c815700 1 --2- 172.21.15.66:0/3512817534 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f8b3402ff60 0x7f8b34032420 secure :-1 s=READY pgs=46 cs=0 l=1 rev1=1 rx=0x7f8b380096a0 tx=0x7f8b38006b40).stop 2023-11-17T05:24:36.046 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:36.045+0000 7f8b4c815700 1 -- 172.21.15.66:0/3512817534 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f8b480f69c0 msgr2=0x7f8b480f4e80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:24:36.047 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:36.045+0000 7f8b4c815700 1 --2- 172.21.15.66:0/3512817534 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f8b480f69c0 0x7f8b480f4e80 secure :-1 s=READY pgs=162 cs=0 l=1 rev1=1 rx=0x7f8b30009f80 tx=0x7f8b30009b30).stop 2023-11-17T05:24:36.047 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:36.045+0000 7f8b4c815700 1 -- 172.21.15.66:0/3512817534 shutdown_connections 2023-11-17T05:24:36.047 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:36.045+0000 7f8b4c815700 1 --2- 172.21.15.66:0/3512817534 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f8b3402ff60 0x7f8b34032420 unknown :-1 s=CLOSED pgs=46 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:24:36.047 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:36.045+0000 7f8b4c815700 1 --2- 172.21.15.66:0/3512817534 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f8b480f69c0 0x7f8b480f4e80 unknown :-1 s=CLOSED pgs=162 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:24:36.047 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:36.045+0000 7f8b4c815700 1 -- 172.21.15.66:0/3512817534 >> 172.21.15.66:0/3512817534 conn(0x7f8b480f09a0 msgr2=0x7f8b480f14e0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:24:36.047 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:36.045+0000 7f8b4c815700 1 -- 172.21.15.66:0/3512817534 shutdown_connections 2023-11-17T05:24:36.047 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:36.045+0000 7f8b4c815700 1 -- 172.21.15.66:0/3512817534 wait complete. 2023-11-17T05:24:36.047 INFO:teuthology.orchestra.run.smithi066.stderr:dumped monmap epoch 1 2023-11-17T05:24:36.739 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:24:36 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:24:35.188566+0000 mgr.smithi040.ogcprl (mgr.14164) 103 : cluster [DBG] pgmap v96: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:24:41.494 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:24:36 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:24:36.045085+0000 mon.smithi040 (mon.0) 160 : audit [DBG] from='client.? 172.21.15.66:0/3512817534' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-11-17T05:24:41.495 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2023-11-17T05:24:41.495 DEBUG:teuthology.orchestra.run.smithi066:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.4 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ed6470e2-8508-11ee-8dbe-212e2dc638e7 -- ceph mon dump -f json 2023-11-17T05:24:41.496 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:24:38 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:24:37.188964+0000 mgr.smithi040.ogcprl (mgr.14164) 104 : cluster [DBG] pgmap v97: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:24:41.496 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:24:40 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:24:39.189225+0000 mgr.smithi040.ogcprl (mgr.14164) 105 : cluster [DBG] pgmap v98: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:24:42.671 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:42.668+0000 7f41c5553700 1 -- 172.21.15.66:0/2933332608 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f41c00f78e0 msgr2=0x7f41c00f7d00 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:24:42.671 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:42.668+0000 7f41c5553700 1 --2- 172.21.15.66:0/2933332608 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f41c00f78e0 0x7f41c00f7d00 secure :-1 s=READY pgs=163 cs=0 l=1 rev1=1 rx=0x7f41b0009fb0 tx=0x7f41b0009b30).stop 2023-11-17T05:24:42.671 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:42.668+0000 7f41c5553700 1 -- 172.21.15.66:0/2933332608 shutdown_connections 2023-11-17T05:24:42.672 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:42.668+0000 7f41c5553700 1 --2- 172.21.15.66:0/2933332608 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f41c00f78e0 0x7f41c00f7d00 unknown :-1 s=CLOSED pgs=163 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:24:42.672 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:42.668+0000 7f41c5553700 1 -- 172.21.15.66:0/2933332608 >> 172.21.15.66:0/2933332608 conn(0x7f41c00f0120 msgr2=0x7f41c00f2540 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:24:42.672 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:42.668+0000 7f41c5553700 1 -- 172.21.15.66:0/2933332608 shutdown_connections 2023-11-17T05:24:42.672 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:42.668+0000 7f41c5553700 1 -- 172.21.15.66:0/2933332608 wait complete. 2023-11-17T05:24:42.672 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:42.668+0000 7f41c5553700 1 Processor -- start 2023-11-17T05:24:42.672 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:42.669+0000 7f41c5553700 1 -- start start 2023-11-17T05:24:42.672 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:42.669+0000 7f41c5553700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f41c00f78e0 0x7f41c01064e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:24:42.672 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:42.669+0000 7f41c5553700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f41c01069d0 con 0x7f41c00f78e0 2023-11-17T05:24:42.672 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:42.669+0000 7f41bffff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f41c00f78e0 0x7f41c01064e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:24:42.672 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:42.669+0000 7f41bffff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f41c00f78e0 0x7f41c01064e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.66:57676/0 (socket says 172.21.15.66:57676) 2023-11-17T05:24:42.672 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:42.669+0000 7f41bffff700 1 -- 172.21.15.66:0/3901943169 learned_addr learned my addr 172.21.15.66:0/3901943169 (peer_addr_for_me v2:172.21.15.66:0/0) 2023-11-17T05:24:42.672 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:42.669+0000 7f41bffff700 1 -- 172.21.15.66:0/3901943169 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f41b0005040 con 0x7f41c00f78e0 2023-11-17T05:24:42.673 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:42.669+0000 7f41bffff700 1 --2- 172.21.15.66:0/3901943169 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f41c00f78e0 0x7f41c01064e0 secure :-1 s=READY pgs=164 cs=0 l=1 rev1=1 rx=0x7f41b0013510 tx=0x7f41b000b040).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:24:42.673 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:42.670+0000 7f41bcff9700 1 -- 172.21.15.66:0/3901943169 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f41b0009320 con 0x7f41c00f78e0 2023-11-17T05:24:42.673 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:42.670+0000 7f41c5553700 1 -- 172.21.15.66:0/3901943169 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f41c0106bd0 con 0x7f41c00f78e0 2023-11-17T05:24:42.673 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:42.670+0000 7f41bcff9700 1 -- 172.21.15.66:0/3901943169 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(25 keys) v1 ==== 1074+0+0 (secure 0 0 0) 0x7f41b0013f00 con 0x7f41c00f78e0 2023-11-17T05:24:42.673 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:42.670+0000 7f41bcff9700 1 -- 172.21.15.66:0/3901943169 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f41b0014dc0 con 0x7f41c00f78e0 2023-11-17T05:24:42.673 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:42.670+0000 7f41c5553700 1 -- 172.21.15.66:0/3901943169 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f41c0107060 con 0x7f41c00f78e0 2023-11-17T05:24:42.674 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:42.670+0000 7f41bcff9700 1 -- 172.21.15.66:0/3901943169 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 17) v1 ==== 36269+0+0 (secure 0 0 0) 0x7f41b00095f0 con 0x7f41c00f78e0 2023-11-17T05:24:42.674 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:42.670+0000 7f41c5553700 1 -- 172.21.15.66:0/3901943169 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f41c0045ad0 con 0x7f41c00f78e0 2023-11-17T05:24:42.674 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:42.671+0000 7f41bcff9700 1 --2- 172.21.15.66:0/3901943169 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f41a802ff60 0x7f41a8032420 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:24:42.674 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:42.671+0000 7f41bcff9700 1 -- 172.21.15.66:0/3901943169 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(5..5 src has 1..5) v4 ==== 1270+0+0 (secure 0 0 0) 0x7f41b0041020 con 0x7f41c00f78e0 2023-11-17T05:24:42.675 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:42.671+0000 7f41bf7fe700 1 --2- 172.21.15.66:0/3901943169 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f41a802ff60 0x7f41a8032420 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:24:42.675 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:42.671+0000 7f41bf7fe700 1 --2- 172.21.15.66:0/3901943169 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f41a802ff60 0x7f41a8032420 secure :-1 s=READY pgs=47 cs=0 l=1 rev1=1 rx=0x7f41b40096a0 tx=0x7f41b4006b40).ready entity=mgr.14164 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:24:42.677 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:42.675+0000 7f41bcff9700 1 -- 172.21.15.66:0/3901943169 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+152212 (secure 0 0 0) 0x7f41b0014420 con 0x7f41c00f78e0 2023-11-17T05:24:42.739 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:24:42 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:24:41.189554+0000 mgr.smithi040.ogcprl (mgr.14164) 106 : cluster [DBG] pgmap v99: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:24:42.896 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:42.894+0000 7f41c5553700 1 -- 172.21.15.66:0/3901943169 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "mon dump", "format": "json"} v 0) v1 -- 0x7f41c0107eb0 con 0x7f41c00f78e0 2023-11-17T05:24:42.896 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:42.895+0000 7f41bcff9700 1 -- 172.21.15.66:0/3901943169 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mon_command_ack([{"prefix": "mon dump", "format": "json"}]=0 dumped monmap epoch 1 v1) v1 ==== 95+0+690 (secure 0 0 0) 0x7f41b001c920 con 0x7f41c00f78e0 2023-11-17T05:24:42.897 INFO:teuthology.orchestra.run.smithi066.stdout: 2023-11-17T05:24:42.897 INFO:teuthology.orchestra.run.smithi066.stdout:{"epoch":1,"fsid":"ed6470e2-8508-11ee-8dbe-212e2dc638e7","modified":"2023-11-17T05:20:13.934038Z","created":"2023-11-17T05:20:13.934038Z","min_mon_release":16,"min_mon_release_name":"pacific","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging"],"optional":[]},"mons":[{"rank":0,"name":"smithi040","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:3300","nonce":0},{"type":"v1","addr":"172.21.15.40:6789","nonce":0}]},"addr":"172.21.15.40:6789/0","public_addr":"172.21.15.40:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2023-11-17T05:24:42.898 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:42.897+0000 7f41c5553700 1 -- 172.21.15.66:0/3901943169 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f41a802ff60 msgr2=0x7f41a8032420 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:24:42.898 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:42.897+0000 7f41c5553700 1 --2- 172.21.15.66:0/3901943169 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f41a802ff60 0x7f41a8032420 secure :-1 s=READY pgs=47 cs=0 l=1 rev1=1 rx=0x7f41b40096a0 tx=0x7f41b4006b40).stop 2023-11-17T05:24:42.898 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:42.897+0000 7f41c5553700 1 -- 172.21.15.66:0/3901943169 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f41c00f78e0 msgr2=0x7f41c01064e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:24:42.898 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:42.897+0000 7f41c5553700 1 --2- 172.21.15.66:0/3901943169 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f41c00f78e0 0x7f41c01064e0 secure :-1 s=READY pgs=164 cs=0 l=1 rev1=1 rx=0x7f41b0013510 tx=0x7f41b000b040).stop 2023-11-17T05:24:42.898 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:42.897+0000 7f41c5553700 1 -- 172.21.15.66:0/3901943169 shutdown_connections 2023-11-17T05:24:42.898 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:42.897+0000 7f41c5553700 1 --2- 172.21.15.66:0/3901943169 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f41a802ff60 0x7f41a8032420 unknown :-1 s=CLOSED pgs=47 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:24:42.899 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:42.897+0000 7f41c5553700 1 --2- 172.21.15.66:0/3901943169 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f41c00f78e0 0x7f41c01064e0 unknown :-1 s=CLOSED pgs=164 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:24:42.899 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:42.897+0000 7f41c5553700 1 -- 172.21.15.66:0/3901943169 >> 172.21.15.66:0/3901943169 conn(0x7f41c00f0120 msgr2=0x7f41c00f0ba0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:24:42.899 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:42.897+0000 7f41c5553700 1 -- 172.21.15.66:0/3901943169 shutdown_connections 2023-11-17T05:24:42.899 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:42.897+0000 7f41c5553700 1 -- 172.21.15.66:0/3901943169 wait complete. 2023-11-17T05:24:42.899 INFO:teuthology.orchestra.run.smithi066.stderr:dumped monmap epoch 1 2023-11-17T05:24:43.739 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:24:43 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:24:42.896625+0000 mon.smithi040 (mon.0) 161 : audit [DBG] from='client.? 172.21.15.66:0/3901943169' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-11-17T05:24:44.288 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2023-11-17T05:24:44.288 DEBUG:teuthology.orchestra.run.smithi066:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.4 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ed6470e2-8508-11ee-8dbe-212e2dc638e7 -- ceph mon dump -f json 2023-11-17T05:24:44.739 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:24:44 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:24:43.189858+0000 mgr.smithi040.ogcprl (mgr.14164) 107 : cluster [DBG] pgmap v100: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:24:45.505 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:45.503+0000 7f3dfb244700 1 -- 172.21.15.66:0/3620754001 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f3df4060280 msgr2=0x7f3df40f90f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:24:45.505 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:45.503+0000 7f3dfb244700 1 --2- 172.21.15.66:0/3620754001 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f3df4060280 0x7f3df40f90f0 secure :-1 s=READY pgs=165 cs=0 l=1 rev1=1 rx=0x7f3de8009fb0 tx=0x7f3de8009b30).stop 2023-11-17T05:24:45.505 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:45.503+0000 7f3dfb244700 1 -- 172.21.15.66:0/3620754001 shutdown_connections 2023-11-17T05:24:45.505 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:45.503+0000 7f3dfb244700 1 --2- 172.21.15.66:0/3620754001 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f3df4060280 0x7f3df40f90f0 unknown :-1 s=CLOSED pgs=165 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:24:45.505 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:45.503+0000 7f3dfb244700 1 -- 172.21.15.66:0/3620754001 >> 172.21.15.66:0/3620754001 conn(0x7f3df40f0b70 msgr2=0x7f3df40f2f90 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:24:45.505 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:45.503+0000 7f3dfb244700 1 -- 172.21.15.66:0/3620754001 shutdown_connections 2023-11-17T05:24:45.505 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:45.503+0000 7f3dfb244700 1 -- 172.21.15.66:0/3620754001 wait complete. 2023-11-17T05:24:45.505 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:45.503+0000 7f3dfb244700 1 Processor -- start 2023-11-17T05:24:45.505 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:45.504+0000 7f3dfb244700 1 -- start start 2023-11-17T05:24:45.506 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:45.504+0000 7f3dfb244700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f3df4060280 0x7f3df41066e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:24:45.508 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:45.504+0000 7f3dfb244700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f3df4106bd0 con 0x7f3df4060280 2023-11-17T05:24:45.508 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:45.504+0000 7f3df8fe0700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f3df4060280 0x7f3df41066e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:24:45.508 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:45.504+0000 7f3df8fe0700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f3df4060280 0x7f3df41066e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.66:57688/0 (socket says 172.21.15.66:57688) 2023-11-17T05:24:45.508 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:45.504+0000 7f3df8fe0700 1 -- 172.21.15.66:0/483878336 learned_addr learned my addr 172.21.15.66:0/483878336 (peer_addr_for_me v2:172.21.15.66:0/0) 2023-11-17T05:24:45.508 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:45.504+0000 7f3df8fe0700 1 -- 172.21.15.66:0/483878336 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f3de8005040 con 0x7f3df4060280 2023-11-17T05:24:45.508 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:45.505+0000 7f3df8fe0700 1 --2- 172.21.15.66:0/483878336 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f3df4060280 0x7f3df41066e0 secure :-1 s=READY pgs=166 cs=0 l=1 rev1=1 rx=0x7f3de8013510 tx=0x7f3de800b040).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:24:45.508 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:45.505+0000 7f3df17fa700 1 -- 172.21.15.66:0/483878336 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f3de8009320 con 0x7f3df4060280 2023-11-17T05:24:45.509 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:45.505+0000 7f3dfb244700 1 -- 172.21.15.66:0/483878336 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f3df4106dd0 con 0x7f3df4060280 2023-11-17T05:24:45.509 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:45.505+0000 7f3dfb244700 1 -- 172.21.15.66:0/483878336 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f3df4107260 con 0x7f3df4060280 2023-11-17T05:24:45.509 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:45.505+0000 7f3df17fa700 1 -- 172.21.15.66:0/483878336 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(25 keys) v1 ==== 1074+0+0 (secure 0 0 0) 0x7f3de8013f00 con 0x7f3df4060280 2023-11-17T05:24:45.509 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:45.505+0000 7f3df17fa700 1 -- 172.21.15.66:0/483878336 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f3de8014cc0 con 0x7f3df4060280 2023-11-17T05:24:45.510 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:45.506+0000 7f3df17fa700 1 -- 172.21.15.66:0/483878336 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 17) v1 ==== 36269+0+0 (secure 0 0 0) 0x7f3de80263a0 con 0x7f3df4060280 2023-11-17T05:24:45.510 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:45.506+0000 7f3df17fa700 1 --2- 172.21.15.66:0/483878336 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f3de402ff60 0x7f3de4032420 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:24:45.510 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:45.506+0000 7f3dfb244700 1 -- 172.21.15.66:0/483878336 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f3dd4004fa0 con 0x7f3df4060280 2023-11-17T05:24:45.510 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:45.506+0000 7f3df17fa700 1 -- 172.21.15.66:0/483878336 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(5..5 src has 1..5) v4 ==== 1270+0+0 (secure 0 0 0) 0x7f3de8040f10 con 0x7f3df4060280 2023-11-17T05:24:45.510 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:45.506+0000 7f3df3fff700 1 --2- 172.21.15.66:0/483878336 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f3de402ff60 0x7f3de4032420 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:24:45.510 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:45.506+0000 7f3df3fff700 1 --2- 172.21.15.66:0/483878336 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f3de402ff60 0x7f3de4032420 secure :-1 s=READY pgs=48 cs=0 l=1 rev1=1 rx=0x7f3de00096a0 tx=0x7f3de0006b40).ready entity=mgr.14164 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:24:45.512 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:45.511+0000 7f3df17fa700 1 -- 172.21.15.66:0/483878336 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+152212 (secure 0 0 0) 0x7f3de8012920 con 0x7f3df4060280 2023-11-17T05:24:45.730 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:45.729+0000 7f3dfb244700 1 -- 172.21.15.66:0/483878336 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "mon dump", "format": "json"} v 0) v1 -- 0x7f3dd4005670 con 0x7f3df4060280 2023-11-17T05:24:45.731 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:45.729+0000 7f3df17fa700 1 -- 172.21.15.66:0/483878336 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mon_command_ack([{"prefix": "mon dump", "format": "json"}]=0 dumped monmap epoch 1 v1) v1 ==== 95+0+690 (secure 0 0 0) 0x7f3de802b030 con 0x7f3df4060280 2023-11-17T05:24:45.731 INFO:teuthology.orchestra.run.smithi066.stdout: 2023-11-17T05:24:45.731 INFO:teuthology.orchestra.run.smithi066.stdout:{"epoch":1,"fsid":"ed6470e2-8508-11ee-8dbe-212e2dc638e7","modified":"2023-11-17T05:20:13.934038Z","created":"2023-11-17T05:20:13.934038Z","min_mon_release":16,"min_mon_release_name":"pacific","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging"],"optional":[]},"mons":[{"rank":0,"name":"smithi040","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:3300","nonce":0},{"type":"v1","addr":"172.21.15.40:6789","nonce":0}]},"addr":"172.21.15.40:6789/0","public_addr":"172.21.15.40:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2023-11-17T05:24:45.732 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:45.731+0000 7f3dfb244700 1 -- 172.21.15.66:0/483878336 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f3de402ff60 msgr2=0x7f3de4032420 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:24:45.733 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:45.731+0000 7f3dfb244700 1 --2- 172.21.15.66:0/483878336 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f3de402ff60 0x7f3de4032420 secure :-1 s=READY pgs=48 cs=0 l=1 rev1=1 rx=0x7f3de00096a0 tx=0x7f3de0006b40).stop 2023-11-17T05:24:45.733 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:45.731+0000 7f3dfb244700 1 -- 172.21.15.66:0/483878336 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f3df4060280 msgr2=0x7f3df41066e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:24:45.733 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:45.731+0000 7f3dfb244700 1 --2- 172.21.15.66:0/483878336 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f3df4060280 0x7f3df41066e0 secure :-1 s=READY pgs=166 cs=0 l=1 rev1=1 rx=0x7f3de8013510 tx=0x7f3de800b040).stop 2023-11-17T05:24:45.733 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:45.731+0000 7f3dfb244700 1 -- 172.21.15.66:0/483878336 shutdown_connections 2023-11-17T05:24:45.733 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:45.731+0000 7f3dfb244700 1 --2- 172.21.15.66:0/483878336 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f3de402ff60 0x7f3de4032420 unknown :-1 s=CLOSED pgs=48 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:24:45.733 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:45.731+0000 7f3dfb244700 1 --2- 172.21.15.66:0/483878336 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f3df4060280 0x7f3df41066e0 unknown :-1 s=CLOSED pgs=166 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:24:45.733 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:45.731+0000 7f3dfb244700 1 -- 172.21.15.66:0/483878336 >> 172.21.15.66:0/483878336 conn(0x7f3df40f0b70 msgr2=0x7f3df40f1660 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:24:45.733 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:45.731+0000 7f3dfb244700 1 -- 172.21.15.66:0/483878336 shutdown_connections 2023-11-17T05:24:45.733 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:45.731+0000 7f3dfb244700 1 -- 172.21.15.66:0/483878336 wait complete. 2023-11-17T05:24:45.733 INFO:teuthology.orchestra.run.smithi066.stderr:dumped monmap epoch 1 2023-11-17T05:24:45.738 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:24:45 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:24:45.190189+0000 mgr.smithi040.ogcprl (mgr.14164) 108 : cluster [DBG] pgmap v101: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:24:46.739 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:24:46 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:24:45.731009+0000 mon.smithi040 (mon.0) 162 : audit [DBG] from='client.? 172.21.15.66:0/483878336' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-11-17T05:24:47.111 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2023-11-17T05:24:47.111 DEBUG:teuthology.orchestra.run.smithi066:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.4 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ed6470e2-8508-11ee-8dbe-212e2dc638e7 -- ceph mon dump -f json 2023-11-17T05:24:47.989 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:24:47 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:24:47.190485+0000 mgr.smithi040.ogcprl (mgr.14164) 109 : cluster [DBG] pgmap v102: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:24:48.315 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:48.313+0000 7f9aa4293700 1 -- 172.21.15.66:0/1093910639 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f9a9c0f5410 msgr2=0x7f9a9c0f5830 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:24:48.315 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:48.313+0000 7f9aa4293700 1 --2- 172.21.15.66:0/1093910639 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f9a9c0f5410 0x7f9a9c0f5830 secure :-1 s=READY pgs=167 cs=0 l=1 rev1=1 rx=0x7f9a98009fb0 tx=0x7f9a98009b30).stop 2023-11-17T05:24:48.315 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:48.313+0000 7f9aa4293700 1 -- 172.21.15.66:0/1093910639 shutdown_connections 2023-11-17T05:24:48.316 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:48.313+0000 7f9aa4293700 1 --2- 172.21.15.66:0/1093910639 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f9a9c0f5410 0x7f9a9c0f5830 unknown :-1 s=CLOSED pgs=167 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:24:48.316 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:48.313+0000 7f9aa4293700 1 -- 172.21.15.66:0/1093910639 >> 172.21.15.66:0/1093910639 conn(0x7f9a9c0f0b70 msgr2=0x7f9a9c0f2f90 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:24:48.316 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:48.313+0000 7f9aa4293700 1 -- 172.21.15.66:0/1093910639 shutdown_connections 2023-11-17T05:24:48.316 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:48.313+0000 7f9aa4293700 1 -- 172.21.15.66:0/1093910639 wait complete. 2023-11-17T05:24:48.316 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:48.314+0000 7f9aa4293700 1 Processor -- start 2023-11-17T05:24:48.316 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:48.314+0000 7f9aa4293700 1 -- start start 2023-11-17T05:24:48.316 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:48.314+0000 7f9aa4293700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f9a9c0f5410 0x7f9a9c109430 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:24:48.316 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:48.314+0000 7f9aa4293700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f9a9c060240 con 0x7f9a9c0f5410 2023-11-17T05:24:48.318 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:48.314+0000 7f9aa202f700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f9a9c0f5410 0x7f9a9c109430 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:24:48.318 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:48.315+0000 7f9aa202f700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f9a9c0f5410 0x7f9a9c109430 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.66:57694/0 (socket says 172.21.15.66:57694) 2023-11-17T05:24:48.318 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:48.315+0000 7f9aa202f700 1 -- 172.21.15.66:0/2696584852 learned_addr learned my addr 172.21.15.66:0/2696584852 (peer_addr_for_me v2:172.21.15.66:0/0) 2023-11-17T05:24:48.318 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:48.315+0000 7f9aa202f700 1 -- 172.21.15.66:0/2696584852 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f9a98005040 con 0x7f9a9c0f5410 2023-11-17T05:24:48.318 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:48.315+0000 7f9aa202f700 1 --2- 172.21.15.66:0/2696584852 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f9a9c0f5410 0x7f9a9c109430 secure :-1 s=READY pgs=168 cs=0 l=1 rev1=1 rx=0x7f9a98013510 tx=0x7f9a9800b040).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:24:48.318 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:48.315+0000 7f9a8effd700 1 -- 172.21.15.66:0/2696584852 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f9a98009320 con 0x7f9a9c0f5410 2023-11-17T05:24:48.318 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:48.315+0000 7f9a8effd700 1 -- 172.21.15.66:0/2696584852 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(25 keys) v1 ==== 1074+0+0 (secure 0 0 0) 0x7f9a98013f00 con 0x7f9a9c0f5410 2023-11-17T05:24:48.318 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:48.315+0000 7f9aa4293700 1 -- 172.21.15.66:0/2696584852 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f9a9c060440 con 0x7f9a9c0f5410 2023-11-17T05:24:48.318 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:48.315+0000 7f9a8effd700 1 -- 172.21.15.66:0/2696584852 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f9a98014d90 con 0x7f9a9c0f5410 2023-11-17T05:24:48.318 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:48.315+0000 7f9aa4293700 1 -- 172.21.15.66:0/2696584852 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f9a9c109ae0 con 0x7f9a9c0f5410 2023-11-17T05:24:48.319 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:48.316+0000 7f9a8effd700 1 -- 172.21.15.66:0/2696584852 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 17) v1 ==== 36269+0+0 (secure 0 0 0) 0x7f9a98014420 con 0x7f9a9c0f5410 2023-11-17T05:24:48.319 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:48.316+0000 7f9aa4293700 1 -- 172.21.15.66:0/2696584852 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f9a9c045ad0 con 0x7f9a9c0f5410 2023-11-17T05:24:48.319 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:48.316+0000 7f9a8effd700 1 --2- 172.21.15.66:0/2696584852 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f9a8802ff60 0x7f9a88032420 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:24:48.319 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:48.316+0000 7f9a8effd700 1 -- 172.21.15.66:0/2696584852 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(5..5 src has 1..5) v4 ==== 1270+0+0 (secure 0 0 0) 0x7f9a98041000 con 0x7f9a9c0f5410 2023-11-17T05:24:48.319 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:48.316+0000 7f9aa182e700 1 --2- 172.21.15.66:0/2696584852 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f9a8802ff60 0x7f9a88032420 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:24:48.319 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:48.317+0000 7f9aa182e700 1 --2- 172.21.15.66:0/2696584852 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f9a8802ff60 0x7f9a88032420 secure :-1 s=READY pgs=49 cs=0 l=1 rev1=1 rx=0x7f9a900096a0 tx=0x7f9a90006b40).ready entity=mgr.14164 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:24:48.323 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:48.321+0000 7f9a8effd700 1 -- 172.21.15.66:0/2696584852 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+152212 (secure 0 0 0) 0x7f9a980095f0 con 0x7f9a9c0f5410 2023-11-17T05:24:48.544 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:48.542+0000 7f9aa4293700 1 -- 172.21.15.66:0/2696584852 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "mon dump", "format": "json"} v 0) v1 -- 0x7f9a9c0fae60 con 0x7f9a9c0f5410 2023-11-17T05:24:48.544 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:48.543+0000 7f9a8effd700 1 -- 172.21.15.66:0/2696584852 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mon_command_ack([{"prefix": "mon dump", "format": "json"}]=0 dumped monmap epoch 1 v1) v1 ==== 95+0+690 (secure 0 0 0) 0x7f9a9801db70 con 0x7f9a9c0f5410 2023-11-17T05:24:48.544 INFO:teuthology.orchestra.run.smithi066.stdout: 2023-11-17T05:24:48.545 INFO:teuthology.orchestra.run.smithi066.stdout:{"epoch":1,"fsid":"ed6470e2-8508-11ee-8dbe-212e2dc638e7","modified":"2023-11-17T05:20:13.934038Z","created":"2023-11-17T05:20:13.934038Z","min_mon_release":16,"min_mon_release_name":"pacific","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging"],"optional":[]},"mons":[{"rank":0,"name":"smithi040","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:3300","nonce":0},{"type":"v1","addr":"172.21.15.40:6789","nonce":0}]},"addr":"172.21.15.40:6789/0","public_addr":"172.21.15.40:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2023-11-17T05:24:48.546 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:48.545+0000 7f9aa4293700 1 -- 172.21.15.66:0/2696584852 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f9a8802ff60 msgr2=0x7f9a88032420 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:24:48.546 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:48.545+0000 7f9aa4293700 1 --2- 172.21.15.66:0/2696584852 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f9a8802ff60 0x7f9a88032420 secure :-1 s=READY pgs=49 cs=0 l=1 rev1=1 rx=0x7f9a900096a0 tx=0x7f9a90006b40).stop 2023-11-17T05:24:48.546 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:48.545+0000 7f9aa4293700 1 -- 172.21.15.66:0/2696584852 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f9a9c0f5410 msgr2=0x7f9a9c109430 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:24:48.546 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:48.545+0000 7f9aa4293700 1 --2- 172.21.15.66:0/2696584852 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f9a9c0f5410 0x7f9a9c109430 secure :-1 s=READY pgs=168 cs=0 l=1 rev1=1 rx=0x7f9a98013510 tx=0x7f9a9800b040).stop 2023-11-17T05:24:48.547 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:48.545+0000 7f9aa4293700 1 -- 172.21.15.66:0/2696584852 shutdown_connections 2023-11-17T05:24:48.547 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:48.545+0000 7f9aa4293700 1 --2- 172.21.15.66:0/2696584852 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f9a8802ff60 0x7f9a88032420 unknown :-1 s=CLOSED pgs=49 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:24:48.547 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:48.545+0000 7f9aa4293700 1 --2- 172.21.15.66:0/2696584852 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f9a9c0f5410 0x7f9a9c109430 unknown :-1 s=CLOSED pgs=168 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:24:48.547 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:48.545+0000 7f9aa4293700 1 -- 172.21.15.66:0/2696584852 >> 172.21.15.66:0/2696584852 conn(0x7f9a9c0f0b70 msgr2=0x7f9a9c0f1660 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:24:48.547 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:48.545+0000 7f9aa4293700 1 -- 172.21.15.66:0/2696584852 shutdown_connections 2023-11-17T05:24:48.547 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:48.545+0000 7f9aa4293700 1 -- 172.21.15.66:0/2696584852 wait complete. 2023-11-17T05:24:48.551 INFO:teuthology.orchestra.run.smithi066.stderr:dumped monmap epoch 1 2023-11-17T05:24:48.989 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:24:48 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:24:48.544718+0000 mon.smithi040 (mon.0) 163 : audit [DBG] from='client.? 172.21.15.66:0/2696584852' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-11-17T05:24:49.932 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2023-11-17T05:24:49.933 DEBUG:teuthology.orchestra.run.smithi066:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.4 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ed6470e2-8508-11ee-8dbe-212e2dc638e7 -- ceph mon dump -f json 2023-11-17T05:24:50.739 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:24:50 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:24:49.190835+0000 mgr.smithi040.ogcprl (mgr.14164) 110 : cluster [DBG] pgmap v103: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:24:51.144 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:51.142+0000 7f01bfe6d700 1 -- 172.21.15.66:0/4153273858 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f01b80f4800 msgr2=0x7f01b80f4c20 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:24:51.144 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:51.142+0000 7f01bfe6d700 1 --2- 172.21.15.66:0/4153273858 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f01b80f4800 0x7f01b80f4c20 secure :-1 s=READY pgs=169 cs=0 l=1 rev1=1 rx=0x7f01b4009fb0 tx=0x7f01b4009b30).stop 2023-11-17T05:24:51.144 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:51.143+0000 7f01bfe6d700 1 -- 172.21.15.66:0/4153273858 shutdown_connections 2023-11-17T05:24:51.144 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:51.143+0000 7f01bfe6d700 1 --2- 172.21.15.66:0/4153273858 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f01b80f4800 0x7f01b80f4c20 unknown :-1 s=CLOSED pgs=169 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:24:51.144 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:51.143+0000 7f01bfe6d700 1 -- 172.21.15.66:0/4153273858 >> 172.21.15.66:0/4153273858 conn(0x7f01b80f0260 msgr2=0x7f01b80f2680 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:24:51.144 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:51.143+0000 7f01bfe6d700 1 -- 172.21.15.66:0/4153273858 shutdown_connections 2023-11-17T05:24:51.145 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:51.143+0000 7f01bfe6d700 1 -- 172.21.15.66:0/4153273858 wait complete. 2023-11-17T05:24:51.146 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:51.143+0000 7f01bfe6d700 1 Processor -- start 2023-11-17T05:24:51.146 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:51.143+0000 7f01bfe6d700 1 -- start start 2023-11-17T05:24:51.146 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:51.143+0000 7f01bfe6d700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f01b80f4800 0x7f01b80f73f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:24:51.146 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:51.143+0000 7f01bfe6d700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f01b80f5870 con 0x7f01b80f4800 2023-11-17T05:24:51.146 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:51.144+0000 7f01bdc09700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f01b80f4800 0x7f01b80f73f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:24:51.146 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:51.144+0000 7f01bdc09700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f01b80f4800 0x7f01b80f73f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.66:57714/0 (socket says 172.21.15.66:57714) 2023-11-17T05:24:51.146 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:51.144+0000 7f01bdc09700 1 -- 172.21.15.66:0/701454355 learned_addr learned my addr 172.21.15.66:0/701454355 (peer_addr_for_me v2:172.21.15.66:0/0) 2023-11-17T05:24:51.146 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:51.144+0000 7f01bdc09700 1 -- 172.21.15.66:0/701454355 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f01b4005040 con 0x7f01b80f4800 2023-11-17T05:24:51.146 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:51.144+0000 7f01bdc09700 1 --2- 172.21.15.66:0/701454355 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f01b80f4800 0x7f01b80f73f0 secure :-1 s=READY pgs=170 cs=0 l=1 rev1=1 rx=0x7f01b4013510 tx=0x7f01b400b040).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:24:51.147 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:51.144+0000 7f01ae7fc700 1 -- 172.21.15.66:0/701454355 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f01b4009320 con 0x7f01b80f4800 2023-11-17T05:24:51.147 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:51.144+0000 7f01ae7fc700 1 -- 172.21.15.66:0/701454355 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(25 keys) v1 ==== 1074+0+0 (secure 0 0 0) 0x7f01b4013f00 con 0x7f01b80f4800 2023-11-17T05:24:51.147 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:51.144+0000 7f01bfe6d700 1 -- 172.21.15.66:0/701454355 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f01b80f5a10 con 0x7f01b80f4800 2023-11-17T05:24:51.147 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:51.144+0000 7f01ae7fc700 1 -- 172.21.15.66:0/701454355 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f01b4014dc0 con 0x7f01b80f4800 2023-11-17T05:24:51.148 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:51.145+0000 7f01bfe6d700 1 -- 172.21.15.66:0/701454355 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f01b80f5ea0 con 0x7f01b80f4800 2023-11-17T05:24:51.148 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:51.145+0000 7f01ae7fc700 1 -- 172.21.15.66:0/701454355 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 17) v1 ==== 36269+0+0 (secure 0 0 0) 0x7f01b401d3d0 con 0x7f01b80f4800 2023-11-17T05:24:51.148 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:51.145+0000 7f01ae7fc700 1 --2- 172.21.15.66:0/701454355 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f01a402ff60 0x7f01a4032420 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:24:51.148 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:51.145+0000 7f01ae7fc700 1 -- 172.21.15.66:0/701454355 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(5..5 src has 1..5) v4 ==== 1270+0+0 (secure 0 0 0) 0x7f01b40410d0 con 0x7f01b80f4800 2023-11-17T05:24:51.148 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:51.146+0000 7f01bd408700 1 --2- 172.21.15.66:0/701454355 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f01a402ff60 0x7f01a4032420 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:24:51.148 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:51.146+0000 7f01bfe6d700 1 -- 172.21.15.66:0/701454355 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f01b8059100 con 0x7f01b80f4800 2023-11-17T05:24:51.148 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:51.146+0000 7f01bd408700 1 --2- 172.21.15.66:0/701454355 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f01a402ff60 0x7f01a4032420 secure :-1 s=READY pgs=50 cs=0 l=1 rev1=1 rx=0x7f01a80096a0 tx=0x7f01a8006b40).ready entity=mgr.14164 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:24:51.152 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:51.151+0000 7f01ae7fc700 1 -- 172.21.15.66:0/701454355 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+152212 (secure 0 0 0) 0x7f01b40095f0 con 0x7f01b80f4800 2023-11-17T05:24:51.370 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:51.369+0000 7f01bfe6d700 1 -- 172.21.15.66:0/701454355 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "mon dump", "format": "json"} v 0) v1 -- 0x7f01b80f6500 con 0x7f01b80f4800 2023-11-17T05:24:51.370 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:51.369+0000 7f01ae7fc700 1 -- 172.21.15.66:0/701454355 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mon_command_ack([{"prefix": "mon dump", "format": "json"}]=0 dumped monmap epoch 1 v1) v1 ==== 95+0+690 (secure 0 0 0) 0x7f01b401cb10 con 0x7f01b80f4800 2023-11-17T05:24:51.371 INFO:teuthology.orchestra.run.smithi066.stdout: 2023-11-17T05:24:51.371 INFO:teuthology.orchestra.run.smithi066.stdout:{"epoch":1,"fsid":"ed6470e2-8508-11ee-8dbe-212e2dc638e7","modified":"2023-11-17T05:20:13.934038Z","created":"2023-11-17T05:20:13.934038Z","min_mon_release":16,"min_mon_release_name":"pacific","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging"],"optional":[]},"mons":[{"rank":0,"name":"smithi040","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:3300","nonce":0},{"type":"v1","addr":"172.21.15.40:6789","nonce":0}]},"addr":"172.21.15.40:6789/0","public_addr":"172.21.15.40:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2023-11-17T05:24:51.372 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:51.371+0000 7f01bfe6d700 1 -- 172.21.15.66:0/701454355 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f01a402ff60 msgr2=0x7f01a4032420 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:24:51.372 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:51.371+0000 7f01bfe6d700 1 --2- 172.21.15.66:0/701454355 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f01a402ff60 0x7f01a4032420 secure :-1 s=READY pgs=50 cs=0 l=1 rev1=1 rx=0x7f01a80096a0 tx=0x7f01a8006b40).stop 2023-11-17T05:24:51.372 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:51.371+0000 7f01bfe6d700 1 -- 172.21.15.66:0/701454355 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f01b80f4800 msgr2=0x7f01b80f73f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:24:51.372 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:51.371+0000 7f01bfe6d700 1 --2- 172.21.15.66:0/701454355 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f01b80f4800 0x7f01b80f73f0 secure :-1 s=READY pgs=170 cs=0 l=1 rev1=1 rx=0x7f01b4013510 tx=0x7f01b400b040).stop 2023-11-17T05:24:51.372 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:51.371+0000 7f01bfe6d700 1 -- 172.21.15.66:0/701454355 shutdown_connections 2023-11-17T05:24:51.373 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:51.371+0000 7f01bfe6d700 1 --2- 172.21.15.66:0/701454355 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f01a402ff60 0x7f01a4032420 unknown :-1 s=CLOSED pgs=50 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:24:51.373 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:51.371+0000 7f01bfe6d700 1 --2- 172.21.15.66:0/701454355 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f01b80f4800 0x7f01b80f73f0 unknown :-1 s=CLOSED pgs=170 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:24:51.373 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:51.371+0000 7f01bfe6d700 1 -- 172.21.15.66:0/701454355 >> 172.21.15.66:0/701454355 conn(0x7f01b80f0260 msgr2=0x7f01b80f0da0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:24:51.373 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:51.371+0000 7f01bfe6d700 1 -- 172.21.15.66:0/701454355 shutdown_connections 2023-11-17T05:24:51.373 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:51.371+0000 7f01bfe6d700 1 -- 172.21.15.66:0/701454355 wait complete. 2023-11-17T05:24:51.373 INFO:teuthology.orchestra.run.smithi066.stderr:dumped monmap epoch 1 2023-11-17T05:24:51.739 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:24:51 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:24:51.370779+0000 mon.smithi040 (mon.0) 164 : audit [DBG] from='client.? 172.21.15.66:0/701454355' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-11-17T05:24:52.739 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:24:52 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:24:51.191167+0000 mgr.smithi040.ogcprl (mgr.14164) 111 : cluster [DBG] pgmap v104: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:24:52.744 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2023-11-17T05:24:52.744 DEBUG:teuthology.orchestra.run.smithi066:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.4 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ed6470e2-8508-11ee-8dbe-212e2dc638e7 -- ceph mon dump -f json 2023-11-17T05:24:53.906 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:53.904+0000 7f8e213a7700 1 -- 172.21.15.66:0/1795462696 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f8e1c0f6510 msgr2=0x7f8e1c0f6930 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:24:53.907 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:53.904+0000 7f8e213a7700 1 --2- 172.21.15.66:0/1795462696 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f8e1c0f6510 0x7f8e1c0f6930 secure :-1 s=READY pgs=171 cs=0 l=1 rev1=1 rx=0x7f8e10009fb0 tx=0x7f8e10009b30).stop 2023-11-17T05:24:53.907 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:53.904+0000 7f8e213a7700 1 -- 172.21.15.66:0/1795462696 shutdown_connections 2023-11-17T05:24:53.907 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:53.904+0000 7f8e213a7700 1 --2- 172.21.15.66:0/1795462696 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f8e1c0f6510 0x7f8e1c0f6930 unknown :-1 s=CLOSED pgs=171 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:24:53.907 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:53.904+0000 7f8e213a7700 1 -- 172.21.15.66:0/1795462696 >> 172.21.15.66:0/1795462696 conn(0x7f8e1c0f1c70 msgr2=0x7f8e1c0f4090 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:24:53.907 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:53.904+0000 7f8e213a7700 1 -- 172.21.15.66:0/1795462696 shutdown_connections 2023-11-17T05:24:53.907 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:53.904+0000 7f8e213a7700 1 -- 172.21.15.66:0/1795462696 wait complete. 2023-11-17T05:24:53.907 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:53.905+0000 7f8e213a7700 1 Processor -- start 2023-11-17T05:24:53.907 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:53.905+0000 7f8e213a7700 1 -- start start 2023-11-17T05:24:53.907 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:53.905+0000 7f8e213a7700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f8e1c0f6510 0x7f8e1c106730 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:24:53.907 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:53.905+0000 7f8e213a7700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f8e1c106c20 con 0x7f8e1c0f6510 2023-11-17T05:24:53.907 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:53.905+0000 7f8e1affd700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f8e1c0f6510 0x7f8e1c106730 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:24:53.907 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:53.905+0000 7f8e1affd700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f8e1c0f6510 0x7f8e1c106730 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.66:52614/0 (socket says 172.21.15.66:52614) 2023-11-17T05:24:53.908 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:53.905+0000 7f8e1affd700 1 -- 172.21.15.66:0/489458751 learned_addr learned my addr 172.21.15.66:0/489458751 (peer_addr_for_me v2:172.21.15.66:0/0) 2023-11-17T05:24:53.908 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:53.905+0000 7f8e1affd700 1 -- 172.21.15.66:0/489458751 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f8e10005040 con 0x7f8e1c0f6510 2023-11-17T05:24:53.909 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:53.906+0000 7f8e1affd700 1 --2- 172.21.15.66:0/489458751 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f8e1c0f6510 0x7f8e1c106730 secure :-1 s=READY pgs=172 cs=0 l=1 rev1=1 rx=0x7f8e10013510 tx=0x7f8e1000b040).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:24:53.909 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:53.906+0000 7f8e037fe700 1 -- 172.21.15.66:0/489458751 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f8e10009320 con 0x7f8e1c0f6510 2023-11-17T05:24:53.909 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:53.906+0000 7f8e213a7700 1 -- 172.21.15.66:0/489458751 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f8e1c106e20 con 0x7f8e1c0f6510 2023-11-17T05:24:53.909 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:53.906+0000 7f8e037fe700 1 -- 172.21.15.66:0/489458751 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(25 keys) v1 ==== 1074+0+0 (secure 0 0 0) 0x7f8e10013f00 con 0x7f8e1c0f6510 2023-11-17T05:24:53.909 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:53.906+0000 7f8e037fe700 1 -- 172.21.15.66:0/489458751 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f8e10014d90 con 0x7f8e1c0f6510 2023-11-17T05:24:53.909 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:53.906+0000 7f8e213a7700 1 -- 172.21.15.66:0/489458751 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f8e1c1072b0 con 0x7f8e1c0f6510 2023-11-17T05:24:53.909 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:53.907+0000 7f8e037fe700 1 -- 172.21.15.66:0/489458751 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 17) v1 ==== 36269+0+0 (secure 0 0 0) 0x7f8e100095f0 con 0x7f8e1c0f6510 2023-11-17T05:24:53.909 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:53.907+0000 7f8e213a7700 1 -- 172.21.15.66:0/489458751 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f8e1c045ad0 con 0x7f8e1c0f6510 2023-11-17T05:24:53.910 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:53.907+0000 7f8e037fe700 1 --2- 172.21.15.66:0/489458751 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f8e0402ff60 0x7f8e04032420 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:24:53.910 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:53.907+0000 7f8e037fe700 1 -- 172.21.15.66:0/489458751 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(5..5 src has 1..5) v4 ==== 1270+0+0 (secure 0 0 0) 0x7f8e10041000 con 0x7f8e1c0f6510 2023-11-17T05:24:53.910 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:53.907+0000 7f8e1a7fc700 1 --2- 172.21.15.66:0/489458751 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f8e0402ff60 0x7f8e04032420 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:24:53.910 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:53.907+0000 7f8e1a7fc700 1 --2- 172.21.15.66:0/489458751 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f8e0402ff60 0x7f8e04032420 secure :-1 s=READY pgs=51 cs=0 l=1 rev1=1 rx=0x7f8e0c0096a0 tx=0x7f8e0c006b40).ready entity=mgr.14164 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:24:53.914 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:53.911+0000 7f8e037fe700 1 -- 172.21.15.66:0/489458751 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+152212 (secure 0 0 0) 0x7f8e10014420 con 0x7f8e1c0f6510 2023-11-17T05:24:54.134 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:54.133+0000 7f8e213a7700 1 -- 172.21.15.66:0/489458751 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "mon dump", "format": "json"} v 0) v1 -- 0x7f8e1c0fbf60 con 0x7f8e1c0f6510 2023-11-17T05:24:54.135 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:54.133+0000 7f8e037fe700 1 -- 172.21.15.66:0/489458751 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mon_command_ack([{"prefix": "mon dump", "format": "json"}]=0 dumped monmap epoch 1 v1) v1 ==== 95+0+690 (secure 0 0 0) 0x7f8e1001d7f0 con 0x7f8e1c0f6510 2023-11-17T05:24:54.135 INFO:teuthology.orchestra.run.smithi066.stdout: 2023-11-17T05:24:54.135 INFO:teuthology.orchestra.run.smithi066.stdout:{"epoch":1,"fsid":"ed6470e2-8508-11ee-8dbe-212e2dc638e7","modified":"2023-11-17T05:20:13.934038Z","created":"2023-11-17T05:20:13.934038Z","min_mon_release":16,"min_mon_release_name":"pacific","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging"],"optional":[]},"mons":[{"rank":0,"name":"smithi040","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:3300","nonce":0},{"type":"v1","addr":"172.21.15.40:6789","nonce":0}]},"addr":"172.21.15.40:6789/0","public_addr":"172.21.15.40:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2023-11-17T05:24:54.136 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:54.135+0000 7f8e213a7700 1 -- 172.21.15.66:0/489458751 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f8e0402ff60 msgr2=0x7f8e04032420 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:24:54.136 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:54.135+0000 7f8e213a7700 1 --2- 172.21.15.66:0/489458751 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f8e0402ff60 0x7f8e04032420 secure :-1 s=READY pgs=51 cs=0 l=1 rev1=1 rx=0x7f8e0c0096a0 tx=0x7f8e0c006b40).stop 2023-11-17T05:24:54.137 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:54.135+0000 7f8e213a7700 1 -- 172.21.15.66:0/489458751 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f8e1c0f6510 msgr2=0x7f8e1c106730 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:24:54.137 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:54.135+0000 7f8e213a7700 1 --2- 172.21.15.66:0/489458751 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f8e1c0f6510 0x7f8e1c106730 secure :-1 s=READY pgs=172 cs=0 l=1 rev1=1 rx=0x7f8e10013510 tx=0x7f8e1000b040).stop 2023-11-17T05:24:54.137 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:54.135+0000 7f8e213a7700 1 -- 172.21.15.66:0/489458751 shutdown_connections 2023-11-17T05:24:54.137 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:54.135+0000 7f8e213a7700 1 --2- 172.21.15.66:0/489458751 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f8e0402ff60 0x7f8e04032420 unknown :-1 s=CLOSED pgs=51 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:24:54.137 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:54.135+0000 7f8e213a7700 1 --2- 172.21.15.66:0/489458751 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f8e1c0f6510 0x7f8e1c106730 unknown :-1 s=CLOSED pgs=172 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:24:54.137 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:54.135+0000 7f8e213a7700 1 -- 172.21.15.66:0/489458751 >> 172.21.15.66:0/489458751 conn(0x7f8e1c0f1c70 msgr2=0x7f8e1c0f2760 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:24:54.137 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:54.135+0000 7f8e213a7700 1 -- 172.21.15.66:0/489458751 shutdown_connections 2023-11-17T05:24:54.137 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:54.135+0000 7f8e213a7700 1 -- 172.21.15.66:0/489458751 wait complete. 2023-11-17T05:24:54.137 INFO:teuthology.orchestra.run.smithi066.stderr:dumped monmap epoch 1 2023-11-17T05:24:54.739 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:24:54 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:24:53.191535+0000 mgr.smithi040.ogcprl (mgr.14164) 112 : cluster [DBG] pgmap v105: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:24:54.739 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:24:54 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:24:54.135043+0000 mon.smithi040 (mon.0) 165 : audit [DBG] from='client.? 172.21.15.66:0/489458751' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-11-17T05:24:55.523 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2023-11-17T05:24:55.523 DEBUG:teuthology.orchestra.run.smithi066:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.4 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ed6470e2-8508-11ee-8dbe-212e2dc638e7 -- ceph mon dump -f json 2023-11-17T05:24:56.663 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:56.662+0000 7f355ffff700 1 -- 172.21.15.66:0/4070525346 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f35600f51a0 msgr2=0x7f35600f55c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:24:56.663 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:56.662+0000 7f355ffff700 1 --2- 172.21.15.66:0/4070525346 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f35600f51a0 0x7f35600f55c0 secure :-1 s=READY pgs=173 cs=0 l=1 rev1=1 rx=0x7f3550009fb0 tx=0x7f3550009b30).stop 2023-11-17T05:24:56.663 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:56.662+0000 7f355ffff700 1 -- 172.21.15.66:0/4070525346 shutdown_connections 2023-11-17T05:24:56.664 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:56.662+0000 7f355ffff700 1 --2- 172.21.15.66:0/4070525346 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f35600f51a0 0x7f35600f55c0 unknown :-1 s=CLOSED pgs=173 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:24:56.664 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:56.662+0000 7f355ffff700 1 -- 172.21.15.66:0/4070525346 >> 172.21.15.66:0/4070525346 conn(0x7f35600f09a0 msgr2=0x7f35600f2dc0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:24:56.664 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:56.662+0000 7f355ffff700 1 -- 172.21.15.66:0/4070525346 shutdown_connections 2023-11-17T05:24:56.664 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:56.662+0000 7f355ffff700 1 -- 172.21.15.66:0/4070525346 wait complete. 2023-11-17T05:24:56.666 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:56.662+0000 7f355ffff700 1 Processor -- start 2023-11-17T05:24:56.666 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:56.662+0000 7f355ffff700 1 -- start start 2023-11-17T05:24:56.666 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:56.663+0000 7f355ffff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f35600f51a0 0x7f35601091d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:24:56.666 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:56.663+0000 7f355ffff700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f3560060220 con 0x7f35600f51a0 2023-11-17T05:24:56.666 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:56.663+0000 7f355effd700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f35600f51a0 0x7f35601091d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:24:56.666 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:56.663+0000 7f355effd700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f35600f51a0 0x7f35601091d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.66:52624/0 (socket says 172.21.15.66:52624) 2023-11-17T05:24:56.667 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:56.663+0000 7f355effd700 1 -- 172.21.15.66:0/1450117543 learned_addr learned my addr 172.21.15.66:0/1450117543 (peer_addr_for_me v2:172.21.15.66:0/0) 2023-11-17T05:24:56.667 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:56.663+0000 7f355effd700 1 -- 172.21.15.66:0/1450117543 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f3550005040 con 0x7f35600f51a0 2023-11-17T05:24:56.667 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:56.663+0000 7f355effd700 1 --2- 172.21.15.66:0/1450117543 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f35600f51a0 0x7f35601091d0 secure :-1 s=READY pgs=174 cs=0 l=1 rev1=1 rx=0x7f3550013510 tx=0x7f355000b040).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:24:56.667 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:56.663+0000 7f35477fe700 1 -- 172.21.15.66:0/1450117543 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f3550009320 con 0x7f35600f51a0 2023-11-17T05:24:56.667 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:56.663+0000 7f35477fe700 1 -- 172.21.15.66:0/1450117543 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(25 keys) v1 ==== 1074+0+0 (secure 0 0 0) 0x7f3550013f00 con 0x7f35600f51a0 2023-11-17T05:24:56.667 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:56.663+0000 7f355ffff700 1 -- 172.21.15.66:0/1450117543 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f3560060420 con 0x7f35600f51a0 2023-11-17T05:24:56.667 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:56.664+0000 7f35477fe700 1 -- 172.21.15.66:0/1450117543 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f3550014dc0 con 0x7f35600f51a0 2023-11-17T05:24:56.667 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:56.664+0000 7f355ffff700 1 -- 172.21.15.66:0/1450117543 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f3560109880 con 0x7f35600f51a0 2023-11-17T05:24:56.667 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:56.664+0000 7f35477fe700 1 -- 172.21.15.66:0/1450117543 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 17) v1 ==== 36269+0+0 (secure 0 0 0) 0x7f35500095f0 con 0x7f35600f51a0 2023-11-17T05:24:56.667 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:56.664+0000 7f355ffff700 1 -- 172.21.15.66:0/1450117543 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f3560059100 con 0x7f35600f51a0 2023-11-17T05:24:56.668 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:56.664+0000 7f35477fe700 1 --2- 172.21.15.66:0/1450117543 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f354802ffb0 0x7f3548032470 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:24:56.668 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:56.665+0000 7f35477fe700 1 -- 172.21.15.66:0/1450117543 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(5..5 src has 1..5) v4 ==== 1270+0+0 (secure 0 0 0) 0x7f3550040fb0 con 0x7f35600f51a0 2023-11-17T05:24:56.668 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:56.665+0000 7f355e7fc700 1 --2- 172.21.15.66:0/1450117543 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f354802ffb0 0x7f3548032470 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:24:56.668 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:56.665+0000 7f355e7fc700 1 --2- 172.21.15.66:0/1450117543 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f354802ffb0 0x7f3548032470 secure :-1 s=READY pgs=52 cs=0 l=1 rev1=1 rx=0x7f35540096a0 tx=0x7f3554006b40).ready entity=mgr.14164 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:24:56.671 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:56.670+0000 7f35477fe700 1 -- 172.21.15.66:0/1450117543 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+152212 (secure 0 0 0) 0x7f355006d2b0 con 0x7f35600f51a0 2023-11-17T05:24:56.739 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:24:56 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:24:55.191896+0000 mgr.smithi040.ogcprl (mgr.14164) 113 : cluster [DBG] pgmap v106: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:24:56.895 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:56.893+0000 7f355ffff700 1 -- 172.21.15.66:0/1450117543 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "mon dump", "format": "json"} v 0) v1 -- 0x7f3560045ad0 con 0x7f35600f51a0 2023-11-17T05:24:56.895 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:56.894+0000 7f35477fe700 1 -- 172.21.15.66:0/1450117543 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mon_command_ack([{"prefix": "mon dump", "format": "json"}]=0 dumped monmap epoch 1 v1) v1 ==== 95+0+690 (secure 0 0 0) 0x7f355001d7f0 con 0x7f35600f51a0 2023-11-17T05:24:56.896 INFO:teuthology.orchestra.run.smithi066.stdout: 2023-11-17T05:24:56.896 INFO:teuthology.orchestra.run.smithi066.stdout:{"epoch":1,"fsid":"ed6470e2-8508-11ee-8dbe-212e2dc638e7","modified":"2023-11-17T05:20:13.934038Z","created":"2023-11-17T05:20:13.934038Z","min_mon_release":16,"min_mon_release_name":"pacific","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging"],"optional":[]},"mons":[{"rank":0,"name":"smithi040","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:3300","nonce":0},{"type":"v1","addr":"172.21.15.40:6789","nonce":0}]},"addr":"172.21.15.40:6789/0","public_addr":"172.21.15.40:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2023-11-17T05:24:56.897 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:56.896+0000 7f355ffff700 1 -- 172.21.15.66:0/1450117543 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f354802ffb0 msgr2=0x7f3548032470 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:24:56.897 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:56.896+0000 7f355ffff700 1 --2- 172.21.15.66:0/1450117543 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f354802ffb0 0x7f3548032470 secure :-1 s=READY pgs=52 cs=0 l=1 rev1=1 rx=0x7f35540096a0 tx=0x7f3554006b40).stop 2023-11-17T05:24:56.897 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:56.896+0000 7f355ffff700 1 -- 172.21.15.66:0/1450117543 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f35600f51a0 msgr2=0x7f35601091d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:24:56.897 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:56.896+0000 7f355ffff700 1 --2- 172.21.15.66:0/1450117543 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f35600f51a0 0x7f35601091d0 secure :-1 s=READY pgs=174 cs=0 l=1 rev1=1 rx=0x7f3550013510 tx=0x7f355000b040).stop 2023-11-17T05:24:56.898 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:56.896+0000 7f355ffff700 1 -- 172.21.15.66:0/1450117543 shutdown_connections 2023-11-17T05:24:56.898 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:56.896+0000 7f355ffff700 1 --2- 172.21.15.66:0/1450117543 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f354802ffb0 0x7f3548032470 unknown :-1 s=CLOSED pgs=52 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:24:56.898 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:56.896+0000 7f355ffff700 1 --2- 172.21.15.66:0/1450117543 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f35600f51a0 0x7f35601091d0 unknown :-1 s=CLOSED pgs=174 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:24:56.898 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:56.896+0000 7f355ffff700 1 -- 172.21.15.66:0/1450117543 >> 172.21.15.66:0/1450117543 conn(0x7f35600f09a0 msgr2=0x7f35600f1460 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:24:56.898 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:56.896+0000 7f355ffff700 1 -- 172.21.15.66:0/1450117543 shutdown_connections 2023-11-17T05:24:56.898 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:56.896+0000 7f355ffff700 1 -- 172.21.15.66:0/1450117543 wait complete. 2023-11-17T05:24:56.898 INFO:teuthology.orchestra.run.smithi066.stderr:dumped monmap epoch 1 2023-11-17T05:24:57.740 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:24:57 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:24:56.895364+0000 mon.smithi040 (mon.0) 166 : audit [DBG] from='client.? 172.21.15.66:0/1450117543' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-11-17T05:24:58.272 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2023-11-17T05:24:58.273 DEBUG:teuthology.orchestra.run.smithi066:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.4 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ed6470e2-8508-11ee-8dbe-212e2dc638e7 -- ceph mon dump -f json 2023-11-17T05:24:58.739 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:24:58 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:24:57.192188+0000 mgr.smithi040.ogcprl (mgr.14164) 114 : cluster [DBG] pgmap v107: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:24:59.508 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:59.505+0000 7fd0e01ea700 1 -- 172.21.15.66:0/4078303766 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fd0d8060280 msgr2=0x7fd0d80f90f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:24:59.508 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:59.505+0000 7fd0e01ea700 1 --2- 172.21.15.66:0/4078303766 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fd0d8060280 0x7fd0d80f90f0 secure :-1 s=READY pgs=175 cs=0 l=1 rev1=1 rx=0x7fd0d4009fb0 tx=0x7fd0d4009b30).stop 2023-11-17T05:24:59.508 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:59.505+0000 7fd0e01ea700 1 -- 172.21.15.66:0/4078303766 shutdown_connections 2023-11-17T05:24:59.508 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:59.505+0000 7fd0e01ea700 1 --2- 172.21.15.66:0/4078303766 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fd0d8060280 0x7fd0d80f90f0 unknown :-1 s=CLOSED pgs=175 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:24:59.508 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:59.505+0000 7fd0e01ea700 1 -- 172.21.15.66:0/4078303766 >> 172.21.15.66:0/4078303766 conn(0x7fd0d80f0b70 msgr2=0x7fd0d80f2f90 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:24:59.508 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:59.505+0000 7fd0e01ea700 1 -- 172.21.15.66:0/4078303766 shutdown_connections 2023-11-17T05:24:59.509 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:59.505+0000 7fd0e01ea700 1 -- 172.21.15.66:0/4078303766 wait complete. 2023-11-17T05:24:59.509 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:59.506+0000 7fd0e01ea700 1 Processor -- start 2023-11-17T05:24:59.509 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:59.506+0000 7fd0e01ea700 1 -- start start 2023-11-17T05:24:59.509 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:59.506+0000 7fd0e01ea700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fd0d8060280 0x7fd0d81066e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:24:59.509 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:59.506+0000 7fd0e01ea700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fd0d8106bd0 con 0x7fd0d8060280 2023-11-17T05:24:59.509 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:59.506+0000 7fd0ddf86700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fd0d8060280 0x7fd0d81066e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:24:59.509 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:59.506+0000 7fd0ddf86700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fd0d8060280 0x7fd0d81066e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.66:52636/0 (socket says 172.21.15.66:52636) 2023-11-17T05:24:59.509 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:59.506+0000 7fd0ddf86700 1 -- 172.21.15.66:0/1306522346 learned_addr learned my addr 172.21.15.66:0/1306522346 (peer_addr_for_me v2:172.21.15.66:0/0) 2023-11-17T05:24:59.509 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:59.506+0000 7fd0ddf86700 1 -- 172.21.15.66:0/1306522346 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fd0d4005040 con 0x7fd0d8060280 2023-11-17T05:24:59.509 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:59.507+0000 7fd0ddf86700 1 --2- 172.21.15.66:0/1306522346 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fd0d8060280 0x7fd0d81066e0 secure :-1 s=READY pgs=176 cs=0 l=1 rev1=1 rx=0x7fd0d4013510 tx=0x7fd0d400b040).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:24:59.509 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:59.507+0000 7fd0ca7fc700 1 -- 172.21.15.66:0/1306522346 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7fd0d4009320 con 0x7fd0d8060280 2023-11-17T05:24:59.509 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:59.507+0000 7fd0ca7fc700 1 -- 172.21.15.66:0/1306522346 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(25 keys) v1 ==== 1074+0+0 (secure 0 0 0) 0x7fd0d4013f00 con 0x7fd0d8060280 2023-11-17T05:24:59.509 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:59.507+0000 7fd0e01ea700 1 -- 172.21.15.66:0/1306522346 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fd0d8106dd0 con 0x7fd0d8060280 2023-11-17T05:24:59.509 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:59.507+0000 7fd0ca7fc700 1 -- 172.21.15.66:0/1306522346 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7fd0d4014dc0 con 0x7fd0d8060280 2023-11-17T05:24:59.510 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:59.507+0000 7fd0e01ea700 1 -- 172.21.15.66:0/1306522346 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fd0d8107260 con 0x7fd0d8060280 2023-11-17T05:24:59.510 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:59.508+0000 7fd0ca7fc700 1 -- 172.21.15.66:0/1306522346 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 17) v1 ==== 36269+0+0 (secure 0 0 0) 0x7fd0d40095f0 con 0x7fd0d8060280 2023-11-17T05:24:59.510 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:59.508+0000 7fd0e01ea700 1 -- 172.21.15.66:0/1306522346 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fd0d8045ad0 con 0x7fd0d8060280 2023-11-17T05:24:59.511 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:59.508+0000 7fd0ca7fc700 1 --2- 172.21.15.66:0/1306522346 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fd0c402ffb0 0x7fd0c4032470 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:24:59.511 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:59.508+0000 7fd0ca7fc700 1 -- 172.21.15.66:0/1306522346 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(5..5 src has 1..5) v4 ==== 1270+0+0 (secure 0 0 0) 0x7fd0d4040ff0 con 0x7fd0d8060280 2023-11-17T05:24:59.511 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:59.508+0000 7fd0dd785700 1 --2- 172.21.15.66:0/1306522346 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fd0c402ffb0 0x7fd0c4032470 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:24:59.511 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:59.508+0000 7fd0dd785700 1 --2- 172.21.15.66:0/1306522346 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fd0c402ffb0 0x7fd0c4032470 secure :-1 s=READY pgs=53 cs=0 l=1 rev1=1 rx=0x7fd0cc0096a0 tx=0x7fd0cc006b40).ready entity=mgr.14164 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:24:59.513 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:59.512+0000 7fd0ca7fc700 1 -- 172.21.15.66:0/1306522346 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+152212 (secure 0 0 0) 0x7fd0d406d2b0 con 0x7fd0d8060280 2023-11-17T05:24:59.732 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:59.731+0000 7fd0e01ea700 1 -- 172.21.15.66:0/1306522346 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "mon dump", "format": "json"} v 0) v1 -- 0x7fd0d8108110 con 0x7fd0d8060280 2023-11-17T05:24:59.733 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:59.731+0000 7fd0ca7fc700 1 -- 172.21.15.66:0/1306522346 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mon_command_ack([{"prefix": "mon dump", "format": "json"}]=0 dumped monmap epoch 1 v1) v1 ==== 95+0+690 (secure 0 0 0) 0x7fd0d4014420 con 0x7fd0d8060280 2023-11-17T05:24:59.733 INFO:teuthology.orchestra.run.smithi066.stdout: 2023-11-17T05:24:59.733 INFO:teuthology.orchestra.run.smithi066.stdout:{"epoch":1,"fsid":"ed6470e2-8508-11ee-8dbe-212e2dc638e7","modified":"2023-11-17T05:20:13.934038Z","created":"2023-11-17T05:20:13.934038Z","min_mon_release":16,"min_mon_release_name":"pacific","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging"],"optional":[]},"mons":[{"rank":0,"name":"smithi040","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:3300","nonce":0},{"type":"v1","addr":"172.21.15.40:6789","nonce":0}]},"addr":"172.21.15.40:6789/0","public_addr":"172.21.15.40:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2023-11-17T05:24:59.736 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:59.733+0000 7fd0e01ea700 1 -- 172.21.15.66:0/1306522346 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fd0c402ffb0 msgr2=0x7fd0c4032470 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:24:59.736 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:59.733+0000 7fd0e01ea700 1 --2- 172.21.15.66:0/1306522346 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fd0c402ffb0 0x7fd0c4032470 secure :-1 s=READY pgs=53 cs=0 l=1 rev1=1 rx=0x7fd0cc0096a0 tx=0x7fd0cc006b40).stop 2023-11-17T05:24:59.736 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:59.733+0000 7fd0e01ea700 1 -- 172.21.15.66:0/1306522346 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fd0d8060280 msgr2=0x7fd0d81066e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:24:59.736 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:59.733+0000 7fd0e01ea700 1 --2- 172.21.15.66:0/1306522346 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fd0d8060280 0x7fd0d81066e0 secure :-1 s=READY pgs=176 cs=0 l=1 rev1=1 rx=0x7fd0d4013510 tx=0x7fd0d400b040).stop 2023-11-17T05:24:59.736 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:59.733+0000 7fd0e01ea700 1 -- 172.21.15.66:0/1306522346 shutdown_connections 2023-11-17T05:24:59.736 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:59.733+0000 7fd0e01ea700 1 --2- 172.21.15.66:0/1306522346 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fd0c402ffb0 0x7fd0c4032470 unknown :-1 s=CLOSED pgs=53 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:24:59.736 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:59.733+0000 7fd0e01ea700 1 --2- 172.21.15.66:0/1306522346 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fd0d8060280 0x7fd0d81066e0 unknown :-1 s=CLOSED pgs=176 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:24:59.737 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:59.733+0000 7fd0e01ea700 1 -- 172.21.15.66:0/1306522346 >> 172.21.15.66:0/1306522346 conn(0x7fd0d80f0b70 msgr2=0x7fd0d80f1660 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:24:59.737 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:59.733+0000 7fd0e01ea700 1 -- 172.21.15.66:0/1306522346 shutdown_connections 2023-11-17T05:24:59.737 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:24:59.733+0000 7fd0e01ea700 1 -- 172.21.15.66:0/1306522346 wait complete. 2023-11-17T05:24:59.737 INFO:teuthology.orchestra.run.smithi066.stderr:dumped monmap epoch 1 2023-11-17T05:25:00.739 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:25:00 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:24:59.192461+0000 mgr.smithi040.ogcprl (mgr.14164) 115 : cluster [DBG] pgmap v108: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:25:00.739 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:25:00 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:24:59.732969+0000 mon.smithi040 (mon.0) 167 : audit [DBG] from='client.? 172.21.15.66:0/1306522346' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-11-17T05:25:01.066 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2023-11-17T05:25:01.066 DEBUG:teuthology.orchestra.run.smithi066:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.4 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ed6470e2-8508-11ee-8dbe-212e2dc638e7 -- ceph mon dump -f json 2023-11-17T05:25:02.251 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:02.249+0000 7f03c77d0700 1 -- 172.21.15.66:0/4157786561 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f03c00f6510 msgr2=0x7f03c00f6930 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:25:02.251 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:02.249+0000 7f03c77d0700 1 --2- 172.21.15.66:0/4157786561 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f03c00f6510 0x7f03c00f6930 secure :-1 s=READY pgs=177 cs=0 l=1 rev1=1 rx=0x7f03b0009fb0 tx=0x7f03b0009b30).stop 2023-11-17T05:25:02.251 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:02.249+0000 7f03c77d0700 1 -- 172.21.15.66:0/4157786561 shutdown_connections 2023-11-17T05:25:02.251 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:02.249+0000 7f03c77d0700 1 --2- 172.21.15.66:0/4157786561 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f03c00f6510 0x7f03c00f6930 unknown :-1 s=CLOSED pgs=177 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:25:02.252 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:02.249+0000 7f03c77d0700 1 -- 172.21.15.66:0/4157786561 >> 172.21.15.66:0/4157786561 conn(0x7f03c00f1c70 msgr2=0x7f03c00f4090 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:25:02.252 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:02.249+0000 7f03c77d0700 1 -- 172.21.15.66:0/4157786561 shutdown_connections 2023-11-17T05:25:02.252 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:02.249+0000 7f03c77d0700 1 -- 172.21.15.66:0/4157786561 wait complete. 2023-11-17T05:25:02.252 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:02.250+0000 7f03c77d0700 1 Processor -- start 2023-11-17T05:25:02.252 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:02.250+0000 7f03c77d0700 1 -- start start 2023-11-17T05:25:02.253 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:02.250+0000 7f03c77d0700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f03c00f6510 0x7f03c0106730 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:25:02.254 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:02.250+0000 7f03c77d0700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f03c0106c20 con 0x7f03c00f6510 2023-11-17T05:25:02.254 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:02.250+0000 7f03c556c700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f03c00f6510 0x7f03c0106730 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:25:02.254 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:02.250+0000 7f03c556c700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f03c00f6510 0x7f03c0106730 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.66:52650/0 (socket says 172.21.15.66:52650) 2023-11-17T05:25:02.254 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:02.250+0000 7f03c556c700 1 -- 172.21.15.66:0/2094484412 learned_addr learned my addr 172.21.15.66:0/2094484412 (peer_addr_for_me v2:172.21.15.66:0/0) 2023-11-17T05:25:02.254 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:02.251+0000 7f03c556c700 1 -- 172.21.15.66:0/2094484412 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f03b0005040 con 0x7f03c00f6510 2023-11-17T05:25:02.254 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:02.251+0000 7f03c556c700 1 --2- 172.21.15.66:0/2094484412 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f03c00f6510 0x7f03c0106730 secure :-1 s=READY pgs=178 cs=0 l=1 rev1=1 rx=0x7f03b0013510 tx=0x7f03b000b040).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:25:02.254 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:02.251+0000 7f03b5ffb700 1 -- 172.21.15.66:0/2094484412 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f03b0009320 con 0x7f03c00f6510 2023-11-17T05:25:02.254 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:02.251+0000 7f03c77d0700 1 -- 172.21.15.66:0/2094484412 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f03c0106e20 con 0x7f03c00f6510 2023-11-17T05:25:02.254 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:02.251+0000 7f03b5ffb700 1 -- 172.21.15.66:0/2094484412 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(25 keys) v1 ==== 1074+0+0 (secure 0 0 0) 0x7f03b0013f00 con 0x7f03c00f6510 2023-11-17T05:25:02.254 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:02.251+0000 7f03b5ffb700 1 -- 172.21.15.66:0/2094484412 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f03b0014d90 con 0x7f03c00f6510 2023-11-17T05:25:02.254 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:02.251+0000 7f03c77d0700 1 -- 172.21.15.66:0/2094484412 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f03c01072b0 con 0x7f03c00f6510 2023-11-17T05:25:02.255 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:02.252+0000 7f03b5ffb700 1 -- 172.21.15.66:0/2094484412 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 17) v1 ==== 36269+0+0 (secure 0 0 0) 0x7f03b00263a0 con 0x7f03c00f6510 2023-11-17T05:25:02.255 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:02.252+0000 7f03b5ffb700 1 --2- 172.21.15.66:0/2094484412 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f03ac02ff60 0x7f03ac032420 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:25:02.255 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:02.252+0000 7f03c77d0700 1 -- 172.21.15.66:0/2094484412 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f03c0045ad0 con 0x7f03c00f6510 2023-11-17T05:25:02.255 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:02.252+0000 7f03b5ffb700 1 -- 172.21.15.66:0/2094484412 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(5..5 src has 1..5) v4 ==== 1270+0+0 (secure 0 0 0) 0x7f03b0041000 con 0x7f03c00f6510 2023-11-17T05:25:02.255 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:02.252+0000 7f03c4d6b700 1 --2- 172.21.15.66:0/2094484412 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f03ac02ff60 0x7f03ac032420 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:25:02.255 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:02.253+0000 7f03c4d6b700 1 --2- 172.21.15.66:0/2094484412 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f03ac02ff60 0x7f03ac032420 secure :-1 s=READY pgs=54 cs=0 l=1 rev1=1 rx=0x7f03bc0096a0 tx=0x7f03bc006b40).ready entity=mgr.14164 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:25:02.258 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:02.257+0000 7f03b5ffb700 1 -- 172.21.15.66:0/2094484412 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+152212 (secure 0 0 0) 0x7f03b0012920 con 0x7f03c00f6510 2023-11-17T05:25:02.480 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:02.478+0000 7f03c77d0700 1 -- 172.21.15.66:0/2094484412 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "mon dump", "format": "json"} v 0) v1 -- 0x7f03c00fc050 con 0x7f03c00f6510 2023-11-17T05:25:02.480 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:02.479+0000 7f03b5ffb700 1 -- 172.21.15.66:0/2094484412 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mon_command_ack([{"prefix": "mon dump", "format": "json"}]=0 dumped monmap epoch 1 v1) v1 ==== 95+0+690 (secure 0 0 0) 0x7f03b002b020 con 0x7f03c00f6510 2023-11-17T05:25:02.481 INFO:teuthology.orchestra.run.smithi066.stdout: 2023-11-17T05:25:02.481 INFO:teuthology.orchestra.run.smithi066.stdout:{"epoch":1,"fsid":"ed6470e2-8508-11ee-8dbe-212e2dc638e7","modified":"2023-11-17T05:20:13.934038Z","created":"2023-11-17T05:20:13.934038Z","min_mon_release":16,"min_mon_release_name":"pacific","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging"],"optional":[]},"mons":[{"rank":0,"name":"smithi040","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:3300","nonce":0},{"type":"v1","addr":"172.21.15.40:6789","nonce":0}]},"addr":"172.21.15.40:6789/0","public_addr":"172.21.15.40:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2023-11-17T05:25:02.482 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:02.481+0000 7f03c77d0700 1 -- 172.21.15.66:0/2094484412 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f03ac02ff60 msgr2=0x7f03ac032420 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:25:02.482 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:02.481+0000 7f03c77d0700 1 --2- 172.21.15.66:0/2094484412 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f03ac02ff60 0x7f03ac032420 secure :-1 s=READY pgs=54 cs=0 l=1 rev1=1 rx=0x7f03bc0096a0 tx=0x7f03bc006b40).stop 2023-11-17T05:25:02.482 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:02.481+0000 7f03c77d0700 1 -- 172.21.15.66:0/2094484412 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f03c00f6510 msgr2=0x7f03c0106730 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:25:02.483 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:02.481+0000 7f03c77d0700 1 --2- 172.21.15.66:0/2094484412 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f03c00f6510 0x7f03c0106730 secure :-1 s=READY pgs=178 cs=0 l=1 rev1=1 rx=0x7f03b0013510 tx=0x7f03b000b040).stop 2023-11-17T05:25:02.483 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:02.481+0000 7f03c77d0700 1 -- 172.21.15.66:0/2094484412 shutdown_connections 2023-11-17T05:25:02.483 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:02.481+0000 7f03c77d0700 1 --2- 172.21.15.66:0/2094484412 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f03ac02ff60 0x7f03ac032420 unknown :-1 s=CLOSED pgs=54 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:25:02.483 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:02.481+0000 7f03c77d0700 1 --2- 172.21.15.66:0/2094484412 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f03c00f6510 0x7f03c0106730 unknown :-1 s=CLOSED pgs=178 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:25:02.483 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:02.481+0000 7f03c77d0700 1 -- 172.21.15.66:0/2094484412 >> 172.21.15.66:0/2094484412 conn(0x7f03c00f1c70 msgr2=0x7f03c00f2760 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:25:02.483 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:02.481+0000 7f03c77d0700 1 -- 172.21.15.66:0/2094484412 shutdown_connections 2023-11-17T05:25:02.483 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:02.481+0000 7f03c77d0700 1 -- 172.21.15.66:0/2094484412 wait complete. 2023-11-17T05:25:02.483 INFO:teuthology.orchestra.run.smithi066.stderr:dumped monmap epoch 1 2023-11-17T05:25:02.739 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:25:02 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:25:01.192775+0000 mgr.smithi040.ogcprl (mgr.14164) 116 : cluster [DBG] pgmap v109: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:25:03.739 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:25:03 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:25:02.480745+0000 mon.smithi040 (mon.0) 168 : audit [DBG] from='client.? 172.21.15.66:0/2094484412' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-11-17T05:25:03.870 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2023-11-17T05:25:03.871 DEBUG:teuthology.orchestra.run.smithi066:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.4 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ed6470e2-8508-11ee-8dbe-212e2dc638e7 -- ceph mon dump -f json 2023-11-17T05:25:04.739 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:25:04 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:25:03.193112+0000 mgr.smithi040.ogcprl (mgr.14164) 117 : cluster [DBG] pgmap v110: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:25:05.020 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:05.018+0000 7fd761212700 1 -- 172.21.15.66:0/3281808800 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fd75c0f5410 msgr2=0x7fd75c0f5830 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:25:05.020 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:05.018+0000 7fd761212700 1 --2- 172.21.15.66:0/3281808800 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fd75c0f5410 0x7fd75c0f5830 secure :-1 s=READY pgs=179 cs=0 l=1 rev1=1 rx=0x7fd74c009fb0 tx=0x7fd74c009b30).stop 2023-11-17T05:25:05.020 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:05.018+0000 7fd761212700 1 -- 172.21.15.66:0/3281808800 shutdown_connections 2023-11-17T05:25:05.020 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:05.018+0000 7fd761212700 1 --2- 172.21.15.66:0/3281808800 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fd75c0f5410 0x7fd75c0f5830 unknown :-1 s=CLOSED pgs=179 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:25:05.020 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:05.018+0000 7fd761212700 1 -- 172.21.15.66:0/3281808800 >> 172.21.15.66:0/3281808800 conn(0x7fd75c0f0b70 msgr2=0x7fd75c0f2f90 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:25:05.020 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:05.018+0000 7fd761212700 1 -- 172.21.15.66:0/3281808800 shutdown_connections 2023-11-17T05:25:05.020 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:05.018+0000 7fd761212700 1 -- 172.21.15.66:0/3281808800 wait complete. 2023-11-17T05:25:05.022 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:05.019+0000 7fd761212700 1 Processor -- start 2023-11-17T05:25:05.022 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:05.019+0000 7fd761212700 1 -- start start 2023-11-17T05:25:05.022 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:05.019+0000 7fd761212700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fd75c0f5410 0x7fd75c105630 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:25:05.022 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:05.019+0000 7fd761212700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fd75c105b20 con 0x7fd75c0f5410 2023-11-17T05:25:05.022 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:05.020+0000 7fd75ad9d700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fd75c0f5410 0x7fd75c105630 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:25:05.022 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:05.020+0000 7fd75ad9d700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fd75c0f5410 0x7fd75c105630 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.66:39458/0 (socket says 172.21.15.66:39458) 2023-11-17T05:25:05.022 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:05.020+0000 7fd75ad9d700 1 -- 172.21.15.66:0/799756260 learned_addr learned my addr 172.21.15.66:0/799756260 (peer_addr_for_me v2:172.21.15.66:0/0) 2023-11-17T05:25:05.022 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:05.020+0000 7fd75ad9d700 1 -- 172.21.15.66:0/799756260 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fd74c005040 con 0x7fd75c0f5410 2023-11-17T05:25:05.023 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:05.020+0000 7fd75ad9d700 1 --2- 172.21.15.66:0/799756260 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fd75c0f5410 0x7fd75c105630 secure :-1 s=READY pgs=180 cs=0 l=1 rev1=1 rx=0x7fd74c013510 tx=0x7fd74c00b040).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:25:05.023 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:05.020+0000 7fd7537fe700 1 -- 172.21.15.66:0/799756260 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7fd74c009320 con 0x7fd75c0f5410 2023-11-17T05:25:05.023 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:05.020+0000 7fd761212700 1 -- 172.21.15.66:0/799756260 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fd75c105d20 con 0x7fd75c0f5410 2023-11-17T05:25:05.023 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:05.021+0000 7fd761212700 1 -- 172.21.15.66:0/799756260 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fd75c1061b0 con 0x7fd75c0f5410 2023-11-17T05:25:05.023 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:05.021+0000 7fd7537fe700 1 -- 172.21.15.66:0/799756260 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(25 keys) v1 ==== 1074+0+0 (secure 0 0 0) 0x7fd74c013f00 con 0x7fd75c0f5410 2023-11-17T05:25:05.023 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:05.021+0000 7fd7537fe700 1 -- 172.21.15.66:0/799756260 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7fd74c014d60 con 0x7fd75c0f5410 2023-11-17T05:25:05.024 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:05.021+0000 7fd7537fe700 1 -- 172.21.15.66:0/799756260 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 17) v1 ==== 36269+0+0 (secure 0 0 0) 0x7fd74c014420 con 0x7fd75c0f5410 2023-11-17T05:25:05.024 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:05.021+0000 7fd761212700 1 -- 172.21.15.66:0/799756260 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fd75c045ad0 con 0x7fd75c0f5410 2023-11-17T05:25:05.024 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:05.021+0000 7fd7537fe700 1 --2- 172.21.15.66:0/799756260 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fd74802ff60 0x7fd748032420 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:25:05.024 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:05.021+0000 7fd7537fe700 1 -- 172.21.15.66:0/799756260 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(5..5 src has 1..5) v4 ==== 1270+0+0 (secure 0 0 0) 0x7fd74c043070 con 0x7fd75c0f5410 2023-11-17T05:25:05.024 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:05.022+0000 7fd75a59c700 1 --2- 172.21.15.66:0/799756260 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fd74802ff60 0x7fd748032420 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:25:05.024 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:05.022+0000 7fd75a59c700 1 --2- 172.21.15.66:0/799756260 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fd74802ff60 0x7fd748032420 secure :-1 s=READY pgs=55 cs=0 l=1 rev1=1 rx=0x7fd7440096a0 tx=0x7fd744006b40).ready entity=mgr.14164 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:25:05.030 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:05.026+0000 7fd7537fe700 1 -- 172.21.15.66:0/799756260 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+152212 (secure 0 0 0) 0x7fd74c012330 con 0x7fd75c0f5410 2023-11-17T05:25:05.249 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:05.248+0000 7fd761212700 1 -- 172.21.15.66:0/799756260 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "mon dump", "format": "json"} v 0) v1 -- 0x7fd75c0fae60 con 0x7fd75c0f5410 2023-11-17T05:25:05.250 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:05.249+0000 7fd7537fe700 1 -- 172.21.15.66:0/799756260 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mon_command_ack([{"prefix": "mon dump", "format": "json"}]=0 dumped monmap epoch 1 v1) v1 ==== 95+0+690 (secure 0 0 0) 0x7fd74c012330 con 0x7fd75c0f5410 2023-11-17T05:25:05.250 INFO:teuthology.orchestra.run.smithi066.stdout: 2023-11-17T05:25:05.250 INFO:teuthology.orchestra.run.smithi066.stdout:{"epoch":1,"fsid":"ed6470e2-8508-11ee-8dbe-212e2dc638e7","modified":"2023-11-17T05:20:13.934038Z","created":"2023-11-17T05:20:13.934038Z","min_mon_release":16,"min_mon_release_name":"pacific","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging"],"optional":[]},"mons":[{"rank":0,"name":"smithi040","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:3300","nonce":0},{"type":"v1","addr":"172.21.15.40:6789","nonce":0}]},"addr":"172.21.15.40:6789/0","public_addr":"172.21.15.40:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2023-11-17T05:25:05.253 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:05.250+0000 7fd761212700 1 -- 172.21.15.66:0/799756260 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fd74802ff60 msgr2=0x7fd748032420 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:25:05.253 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:05.250+0000 7fd761212700 1 --2- 172.21.15.66:0/799756260 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fd74802ff60 0x7fd748032420 secure :-1 s=READY pgs=55 cs=0 l=1 rev1=1 rx=0x7fd7440096a0 tx=0x7fd744006b40).stop 2023-11-17T05:25:05.253 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:05.250+0000 7fd761212700 1 -- 172.21.15.66:0/799756260 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fd75c0f5410 msgr2=0x7fd75c105630 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:25:05.253 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:05.250+0000 7fd761212700 1 --2- 172.21.15.66:0/799756260 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fd75c0f5410 0x7fd75c105630 secure :-1 s=READY pgs=180 cs=0 l=1 rev1=1 rx=0x7fd74c013510 tx=0x7fd74c00b040).stop 2023-11-17T05:25:05.253 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:05.250+0000 7fd761212700 1 -- 172.21.15.66:0/799756260 shutdown_connections 2023-11-17T05:25:05.253 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:05.250+0000 7fd761212700 1 --2- 172.21.15.66:0/799756260 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fd74802ff60 0x7fd748032420 unknown :-1 s=CLOSED pgs=55 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:25:05.253 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:05.250+0000 7fd761212700 1 --2- 172.21.15.66:0/799756260 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fd75c0f5410 0x7fd75c105630 unknown :-1 s=CLOSED pgs=180 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:25:05.253 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:05.250+0000 7fd761212700 1 -- 172.21.15.66:0/799756260 >> 172.21.15.66:0/799756260 conn(0x7fd75c0f0b70 msgr2=0x7fd75c0f1660 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:25:05.253 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:05.250+0000 7fd761212700 1 -- 172.21.15.66:0/799756260 shutdown_connections 2023-11-17T05:25:05.254 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:05.250+0000 7fd761212700 1 -- 172.21.15.66:0/799756260 wait complete. 2023-11-17T05:25:05.254 INFO:teuthology.orchestra.run.smithi066.stderr:dumped monmap epoch 1 2023-11-17T05:25:05.738 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:25:05 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:25:05.250262+0000 mon.smithi040 (mon.0) 169 : audit [DBG] from='client.? 172.21.15.66:0/799756260' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-11-17T05:25:06.688 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2023-11-17T05:25:06.688 DEBUG:teuthology.orchestra.run.smithi066:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.4 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ed6470e2-8508-11ee-8dbe-212e2dc638e7 -- ceph mon dump -f json 2023-11-17T05:25:06.739 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:25:06 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:25:05.193438+0000 mgr.smithi040.ogcprl (mgr.14164) 118 : cluster [DBG] pgmap v111: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:25:07.918 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:07.916+0000 7f58a9530700 1 -- 172.21.15.66:0/4090882375 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f58a40601e0 msgr2=0x7f58a40f8f20 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:25:07.918 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:07.916+0000 7f58a9530700 1 --2- 172.21.15.66:0/4090882375 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f58a40601e0 0x7f58a40f8f20 secure :-1 s=READY pgs=181 cs=0 l=1 rev1=1 rx=0x7f5894009fb0 tx=0x7f5894009b30).stop 2023-11-17T05:25:07.919 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:07.917+0000 7f58a9530700 1 -- 172.21.15.66:0/4090882375 shutdown_connections 2023-11-17T05:25:07.919 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:07.917+0000 7f58a9530700 1 --2- 172.21.15.66:0/4090882375 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f58a40601e0 0x7f58a40f8f20 unknown :-1 s=CLOSED pgs=181 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:25:07.919 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:07.917+0000 7f58a9530700 1 -- 172.21.15.66:0/4090882375 >> 172.21.15.66:0/4090882375 conn(0x7f58a40f09a0 msgr2=0x7f58a40f2dc0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:25:07.919 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:07.917+0000 7f58a9530700 1 -- 172.21.15.66:0/4090882375 shutdown_connections 2023-11-17T05:25:07.919 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:07.917+0000 7f58a9530700 1 -- 172.21.15.66:0/4090882375 wait complete. 2023-11-17T05:25:07.919 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:07.917+0000 7f58a9530700 1 Processor -- start 2023-11-17T05:25:07.919 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:07.917+0000 7f58a9530700 1 -- start start 2023-11-17T05:25:07.921 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:07.917+0000 7f58a9530700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f58a40601e0 0x7f58a4106500 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:25:07.921 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:07.917+0000 7f58a9530700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f58a41069f0 con 0x7f58a40601e0 2023-11-17T05:25:07.921 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:07.918+0000 7f58a3fff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f58a40601e0 0x7f58a4106500 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:25:07.921 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:07.918+0000 7f58a3fff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f58a40601e0 0x7f58a4106500 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.66:39464/0 (socket says 172.21.15.66:39464) 2023-11-17T05:25:07.921 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:07.918+0000 7f58a3fff700 1 -- 172.21.15.66:0/4153604015 learned_addr learned my addr 172.21.15.66:0/4153604015 (peer_addr_for_me v2:172.21.15.66:0/0) 2023-11-17T05:25:07.921 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:07.918+0000 7f58a3fff700 1 -- 172.21.15.66:0/4153604015 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f5894005040 con 0x7f58a40601e0 2023-11-17T05:25:07.921 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:07.918+0000 7f58a3fff700 1 --2- 172.21.15.66:0/4153604015 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f58a40601e0 0x7f58a4106500 secure :-1 s=READY pgs=182 cs=0 l=1 rev1=1 rx=0x7f5894013510 tx=0x7f589400b040).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:25:07.922 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:07.918+0000 7f58a0ff9700 1 -- 172.21.15.66:0/4153604015 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f5894009320 con 0x7f58a40601e0 2023-11-17T05:25:07.922 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:07.918+0000 7f58a9530700 1 -- 172.21.15.66:0/4153604015 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f58a4106bf0 con 0x7f58a40601e0 2023-11-17T05:25:07.922 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:07.918+0000 7f58a0ff9700 1 -- 172.21.15.66:0/4153604015 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(25 keys) v1 ==== 1074+0+0 (secure 0 0 0) 0x7f5894013f00 con 0x7f58a40601e0 2023-11-17T05:25:07.922 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:07.918+0000 7f58a0ff9700 1 -- 172.21.15.66:0/4153604015 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f5894014d90 con 0x7f58a40601e0 2023-11-17T05:25:07.922 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:07.918+0000 7f58a9530700 1 -- 172.21.15.66:0/4153604015 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f58a4107080 con 0x7f58a40601e0 2023-11-17T05:25:07.922 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:07.919+0000 7f58a0ff9700 1 -- 172.21.15.66:0/4153604015 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 17) v1 ==== 36269+0+0 (secure 0 0 0) 0x7f58940095f0 con 0x7f58a40601e0 2023-11-17T05:25:07.922 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:07.919+0000 7f58a9530700 1 -- 172.21.15.66:0/4153604015 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f58a4045ad0 con 0x7f58a40601e0 2023-11-17T05:25:07.922 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:07.919+0000 7f58a0ff9700 1 --2- 172.21.15.66:0/4153604015 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f588c02ff60 0x7f588c032420 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:25:07.922 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:07.919+0000 7f58a0ff9700 1 -- 172.21.15.66:0/4153604015 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(5..5 src has 1..5) v4 ==== 1270+0+0 (secure 0 0 0) 0x7f5894041080 con 0x7f58a40601e0 2023-11-17T05:25:07.922 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:07.920+0000 7f58a37fe700 1 --2- 172.21.15.66:0/4153604015 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f588c02ff60 0x7f588c032420 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:25:07.923 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:07.920+0000 7f58a37fe700 1 --2- 172.21.15.66:0/4153604015 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f588c02ff60 0x7f588c032420 secure :-1 s=READY pgs=56 cs=0 l=1 rev1=1 rx=0x7f58980096a0 tx=0x7f5898006b40).ready entity=mgr.14164 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:25:07.926 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:07.924+0000 7f58a0ff9700 1 -- 172.21.15.66:0/4153604015 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+152212 (secure 0 0 0) 0x7f5894014420 con 0x7f58a40601e0 2023-11-17T05:25:08.144 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:08.143+0000 7f58a9530700 1 -- 172.21.15.66:0/4153604015 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "mon dump", "format": "json"} v 0) v1 -- 0x7f58a4107f30 con 0x7f58a40601e0 2023-11-17T05:25:08.145 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:08.144+0000 7f58a0ff9700 1 -- 172.21.15.66:0/4153604015 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mon_command_ack([{"prefix": "mon dump", "format": "json"}]=0 dumped monmap epoch 1 v1) v1 ==== 95+0+690 (secure 0 0 0) 0x7f5894010230 con 0x7f58a40601e0 2023-11-17T05:25:08.145 INFO:teuthology.orchestra.run.smithi066.stdout: 2023-11-17T05:25:08.145 INFO:teuthology.orchestra.run.smithi066.stdout:{"epoch":1,"fsid":"ed6470e2-8508-11ee-8dbe-212e2dc638e7","modified":"2023-11-17T05:20:13.934038Z","created":"2023-11-17T05:20:13.934038Z","min_mon_release":16,"min_mon_release_name":"pacific","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging"],"optional":[]},"mons":[{"rank":0,"name":"smithi040","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:3300","nonce":0},{"type":"v1","addr":"172.21.15.40:6789","nonce":0}]},"addr":"172.21.15.40:6789/0","public_addr":"172.21.15.40:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2023-11-17T05:25:08.146 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:08.145+0000 7f58a9530700 1 -- 172.21.15.66:0/4153604015 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f588c02ff60 msgr2=0x7f588c032420 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:25:08.146 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:08.145+0000 7f58a9530700 1 --2- 172.21.15.66:0/4153604015 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f588c02ff60 0x7f588c032420 secure :-1 s=READY pgs=56 cs=0 l=1 rev1=1 rx=0x7f58980096a0 tx=0x7f5898006b40).stop 2023-11-17T05:25:08.146 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:08.145+0000 7f58a9530700 1 -- 172.21.15.66:0/4153604015 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f58a40601e0 msgr2=0x7f58a4106500 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:25:08.146 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:08.145+0000 7f58a9530700 1 --2- 172.21.15.66:0/4153604015 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f58a40601e0 0x7f58a4106500 secure :-1 s=READY pgs=182 cs=0 l=1 rev1=1 rx=0x7f5894013510 tx=0x7f589400b040).stop 2023-11-17T05:25:08.147 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:08.145+0000 7f58a9530700 1 -- 172.21.15.66:0/4153604015 shutdown_connections 2023-11-17T05:25:08.147 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:08.145+0000 7f58a9530700 1 --2- 172.21.15.66:0/4153604015 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f588c02ff60 0x7f588c032420 unknown :-1 s=CLOSED pgs=56 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:25:08.147 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:08.145+0000 7f58a9530700 1 --2- 172.21.15.66:0/4153604015 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f58a40601e0 0x7f58a4106500 unknown :-1 s=CLOSED pgs=182 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:25:08.147 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:08.145+0000 7f58a9530700 1 -- 172.21.15.66:0/4153604015 >> 172.21.15.66:0/4153604015 conn(0x7f58a40f09a0 msgr2=0x7f58a40f1420 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:25:08.147 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:08.145+0000 7f58a9530700 1 -- 172.21.15.66:0/4153604015 shutdown_connections 2023-11-17T05:25:08.147 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:08.145+0000 7f58a9530700 1 -- 172.21.15.66:0/4153604015 wait complete. 2023-11-17T05:25:08.147 INFO:teuthology.orchestra.run.smithi066.stderr:dumped monmap epoch 1 2023-11-17T05:25:08.739 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:25:08 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:25:07.193727+0000 mgr.smithi040.ogcprl (mgr.14164) 119 : cluster [DBG] pgmap v112: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:25:08.739 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:25:08 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:25:08.145211+0000 mon.smithi040 (mon.0) 170 : audit [DBG] from='client.? 172.21.15.66:0/4153604015' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-11-17T05:25:09.532 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2023-11-17T05:25:09.542 DEBUG:teuthology.orchestra.run.smithi066:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.4 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ed6470e2-8508-11ee-8dbe-212e2dc638e7 -- ceph mon dump -f json 2023-11-17T05:25:10.734 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:10.731+0000 7f78e6db9700 1 -- 172.21.15.66:0/2150690826 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f78e00f6510 msgr2=0x7f78e00f6930 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:25:10.734 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:10.731+0000 7f78e6db9700 1 --2- 172.21.15.66:0/2150690826 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f78e00f6510 0x7f78e00f6930 secure :-1 s=READY pgs=183 cs=0 l=1 rev1=1 rx=0x7f78d4009fb0 tx=0x7f78d4009b30).stop 2023-11-17T05:25:10.734 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:10.731+0000 7f78e6db9700 1 -- 172.21.15.66:0/2150690826 shutdown_connections 2023-11-17T05:25:10.734 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:10.731+0000 7f78e6db9700 1 --2- 172.21.15.66:0/2150690826 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f78e00f6510 0x7f78e00f6930 unknown :-1 s=CLOSED pgs=183 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:25:10.734 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:10.731+0000 7f78e6db9700 1 -- 172.21.15.66:0/2150690826 >> 172.21.15.66:0/2150690826 conn(0x7f78e00f1c70 msgr2=0x7f78e00f4090 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:25:10.734 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:10.732+0000 7f78e6db9700 1 -- 172.21.15.66:0/2150690826 shutdown_connections 2023-11-17T05:25:10.734 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:10.732+0000 7f78e6db9700 1 -- 172.21.15.66:0/2150690826 wait complete. 2023-11-17T05:25:10.734 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:10.732+0000 7f78e6db9700 1 Processor -- start 2023-11-17T05:25:10.734 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:10.732+0000 7f78e6db9700 1 -- start start 2023-11-17T05:25:10.735 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:10.732+0000 7f78e6db9700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f78e00f6510 0x7f78e006b9a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:25:10.735 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:10.732+0000 7f78e6db9700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f78e0069e20 con 0x7f78e00f6510 2023-11-17T05:25:10.735 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:10.733+0000 7f78e4b55700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f78e00f6510 0x7f78e006b9a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:25:10.735 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:10.733+0000 7f78e4b55700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f78e00f6510 0x7f78e006b9a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.66:39492/0 (socket says 172.21.15.66:39492) 2023-11-17T05:25:10.735 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:10.733+0000 7f78e4b55700 1 -- 172.21.15.66:0/2550249176 learned_addr learned my addr 172.21.15.66:0/2550249176 (peer_addr_for_me v2:172.21.15.66:0/0) 2023-11-17T05:25:10.736 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:10.733+0000 7f78e4b55700 1 -- 172.21.15.66:0/2550249176 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f78d4005040 con 0x7f78e00f6510 2023-11-17T05:25:10.736 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:10.733+0000 7f78e4b55700 1 --2- 172.21.15.66:0/2550249176 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f78e00f6510 0x7f78e006b9a0 secure :-1 s=READY pgs=184 cs=0 l=1 rev1=1 rx=0x7f78d4013510 tx=0x7f78d400b040).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:25:10.736 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:10.733+0000 7f78dd7fa700 1 -- 172.21.15.66:0/2550249176 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f78d4009320 con 0x7f78e00f6510 2023-11-17T05:25:10.736 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:10.733+0000 7f78e6db9700 1 -- 172.21.15.66:0/2550249176 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f78e0069fc0 con 0x7f78e00f6510 2023-11-17T05:25:10.737 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:10.733+0000 7f78dd7fa700 1 -- 172.21.15.66:0/2550249176 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(25 keys) v1 ==== 1074+0+0 (secure 0 0 0) 0x7f78d4013f00 con 0x7f78e00f6510 2023-11-17T05:25:10.737 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:10.733+0000 7f78dd7fa700 1 -- 172.21.15.66:0/2550249176 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f78d4014d90 con 0x7f78e00f6510 2023-11-17T05:25:10.737 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:10.733+0000 7f78e6db9700 1 -- 172.21.15.66:0/2550249176 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f78e006a450 con 0x7f78e00f6510 2023-11-17T05:25:10.737 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:10.734+0000 7f78dd7fa700 1 -- 172.21.15.66:0/2550249176 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 17) v1 ==== 36269+0+0 (secure 0 0 0) 0x7f78d40263a0 con 0x7f78e00f6510 2023-11-17T05:25:10.737 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:10.734+0000 7f78dd7fa700 1 --2- 172.21.15.66:0/2550249176 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f78d002ff60 0x7f78d0032420 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:25:10.737 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:10.734+0000 7f78dd7fa700 1 -- 172.21.15.66:0/2550249176 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(5..5 src has 1..5) v4 ==== 1270+0+0 (secure 0 0 0) 0x7f78d40410c0 con 0x7f78e00f6510 2023-11-17T05:25:10.737 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:10.734+0000 7f78e6db9700 1 -- 172.21.15.66:0/2550249176 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f78e0059100 con 0x7f78e00f6510 2023-11-17T05:25:10.737 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:10.734+0000 7f78dffff700 1 --2- 172.21.15.66:0/2550249176 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f78d002ff60 0x7f78d0032420 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:25:10.737 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:10.735+0000 7f78dffff700 1 --2- 172.21.15.66:0/2550249176 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f78d002ff60 0x7f78d0032420 secure :-1 s=READY pgs=57 cs=0 l=1 rev1=1 rx=0x7f78cc0096a0 tx=0x7f78cc006b40).ready entity=mgr.14164 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:25:10.738 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:25:10 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:25:09.194064+0000 mgr.smithi040.ogcprl (mgr.14164) 120 : cluster [DBG] pgmap v113: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:25:10.741 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:10.739+0000 7f78dd7fa700 1 -- 172.21.15.66:0/2550249176 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+152212 (secure 0 0 0) 0x7f78d4012920 con 0x7f78e00f6510 2023-11-17T05:25:10.963 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:10.962+0000 7f78e6db9700 1 -- 172.21.15.66:0/2550249176 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "mon dump", "format": "json"} v 0) v1 -- 0x7f78e006ab50 con 0x7f78e00f6510 2023-11-17T05:25:10.964 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:10.962+0000 7f78dd7fa700 1 -- 172.21.15.66:0/2550249176 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mon_command_ack([{"prefix": "mon dump", "format": "json"}]=0 dumped monmap epoch 1 v1) v1 ==== 95+0+690 (secure 0 0 0) 0x7f78d402b020 con 0x7f78e00f6510 2023-11-17T05:25:10.964 INFO:teuthology.orchestra.run.smithi066.stdout: 2023-11-17T05:25:10.964 INFO:teuthology.orchestra.run.smithi066.stdout:{"epoch":1,"fsid":"ed6470e2-8508-11ee-8dbe-212e2dc638e7","modified":"2023-11-17T05:20:13.934038Z","created":"2023-11-17T05:20:13.934038Z","min_mon_release":16,"min_mon_release_name":"pacific","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging"],"optional":[]},"mons":[{"rank":0,"name":"smithi040","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:3300","nonce":0},{"type":"v1","addr":"172.21.15.40:6789","nonce":0}]},"addr":"172.21.15.40:6789/0","public_addr":"172.21.15.40:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2023-11-17T05:25:10.965 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:10.964+0000 7f78e6db9700 1 -- 172.21.15.66:0/2550249176 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f78d002ff60 msgr2=0x7f78d0032420 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:25:10.965 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:10.964+0000 7f78e6db9700 1 --2- 172.21.15.66:0/2550249176 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f78d002ff60 0x7f78d0032420 secure :-1 s=READY pgs=57 cs=0 l=1 rev1=1 rx=0x7f78cc0096a0 tx=0x7f78cc006b40).stop 2023-11-17T05:25:10.966 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:10.964+0000 7f78e6db9700 1 -- 172.21.15.66:0/2550249176 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f78e00f6510 msgr2=0x7f78e006b9a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:25:10.966 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:10.964+0000 7f78e6db9700 1 --2- 172.21.15.66:0/2550249176 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f78e00f6510 0x7f78e006b9a0 secure :-1 s=READY pgs=184 cs=0 l=1 rev1=1 rx=0x7f78d4013510 tx=0x7f78d400b040).stop 2023-11-17T05:25:10.966 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:10.964+0000 7f78e6db9700 1 -- 172.21.15.66:0/2550249176 shutdown_connections 2023-11-17T05:25:10.966 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:10.964+0000 7f78e6db9700 1 --2- 172.21.15.66:0/2550249176 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f78d002ff60 0x7f78d0032420 unknown :-1 s=CLOSED pgs=57 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:25:10.966 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:10.964+0000 7f78e6db9700 1 --2- 172.21.15.66:0/2550249176 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f78e00f6510 0x7f78e006b9a0 unknown :-1 s=CLOSED pgs=184 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:25:10.966 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:10.964+0000 7f78e6db9700 1 -- 172.21.15.66:0/2550249176 >> 172.21.15.66:0/2550249176 conn(0x7f78e00f1c70 msgr2=0x7f78e00f2760 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:25:10.966 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:10.964+0000 7f78e6db9700 1 -- 172.21.15.66:0/2550249176 shutdown_connections 2023-11-17T05:25:10.966 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:10.964+0000 7f78e6db9700 1 -- 172.21.15.66:0/2550249176 wait complete. 2023-11-17T05:25:10.967 INFO:teuthology.orchestra.run.smithi066.stderr:dumped monmap epoch 1 2023-11-17T05:25:11.739 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:25:11 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:25:10.964111+0000 mon.smithi040 (mon.0) 171 : audit [DBG] from='client.? 172.21.15.66:0/2550249176' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-11-17T05:25:11.739 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:25:11 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:25:11.310581+0000 mon.smithi040 (mon.0) 172 : audit [INF] from='mgr.14164 172.21.15.40:0/2203032451' entity='mgr.smithi040.ogcprl' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi040.ogcprl/mirror_snapshot_schedule"}]: dispatch 2023-11-17T05:25:11.739 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:25:11 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:25:11.312927+0000 mon.smithi040 (mon.0) 173 : audit [INF] from='mgr.14164 172.21.15.40:0/2203032451' entity='mgr.smithi040.ogcprl' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi040.ogcprl/trash_purge_schedule"}]: dispatch 2023-11-17T05:25:12.383 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2023-11-17T05:25:12.383 DEBUG:teuthology.orchestra.run.smithi066:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.4 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ed6470e2-8508-11ee-8dbe-212e2dc638e7 -- ceph mon dump -f json 2023-11-17T05:25:12.739 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:25:12 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:25:11.194436+0000 mgr.smithi040.ogcprl (mgr.14164) 121 : cluster [DBG] pgmap v114: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:25:13.570 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:13.561+0000 7f146a2fe700 1 -- 172.21.15.66:0/192720400 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f14640f6510 msgr2=0x7f14640f6930 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:25:13.570 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:13.561+0000 7f146a2fe700 1 --2- 172.21.15.66:0/192720400 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f14640f6510 0x7f14640f6930 secure :-1 s=READY pgs=185 cs=0 l=1 rev1=1 rx=0x7f1454009fb0 tx=0x7f1454009b30).stop 2023-11-17T05:25:13.570 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:13.561+0000 7f146a2fe700 1 -- 172.21.15.66:0/192720400 shutdown_connections 2023-11-17T05:25:13.570 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:13.561+0000 7f146a2fe700 1 --2- 172.21.15.66:0/192720400 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f14640f6510 0x7f14640f6930 unknown :-1 s=CLOSED pgs=185 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:25:13.571 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:13.561+0000 7f146a2fe700 1 -- 172.21.15.66:0/192720400 >> 172.21.15.66:0/192720400 conn(0x7f14640f1c70 msgr2=0x7f14640f4090 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:25:13.571 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:13.561+0000 7f146a2fe700 1 -- 172.21.15.66:0/192720400 shutdown_connections 2023-11-17T05:25:13.571 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:13.561+0000 7f146a2fe700 1 -- 172.21.15.66:0/192720400 wait complete. 2023-11-17T05:25:13.571 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:13.561+0000 7f146a2fe700 1 Processor -- start 2023-11-17T05:25:13.571 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:13.561+0000 7f146a2fe700 1 -- start start 2023-11-17T05:25:13.571 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:13.562+0000 7f146a2fe700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f14640f6510 0x7f1464106730 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:25:13.571 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:13.562+0000 7f146a2fe700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f1464106c20 con 0x7f14640f6510 2023-11-17T05:25:13.571 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:13.562+0000 7f1463fff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f14640f6510 0x7f1464106730 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:25:13.571 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:13.562+0000 7f1463fff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f14640f6510 0x7f1464106730 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.66:40392/0 (socket says 172.21.15.66:40392) 2023-11-17T05:25:13.571 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:13.562+0000 7f1463fff700 1 -- 172.21.15.66:0/1684177134 learned_addr learned my addr 172.21.15.66:0/1684177134 (peer_addr_for_me v2:172.21.15.66:0/0) 2023-11-17T05:25:13.571 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:13.562+0000 7f1463fff700 1 -- 172.21.15.66:0/1684177134 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f1454005040 con 0x7f14640f6510 2023-11-17T05:25:13.571 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:13.562+0000 7f1463fff700 1 --2- 172.21.15.66:0/1684177134 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f14640f6510 0x7f1464106730 secure :-1 s=READY pgs=186 cs=0 l=1 rev1=1 rx=0x7f1454013510 tx=0x7f145400b040).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:25:13.571 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:13.562+0000 7f1460ff9700 1 -- 172.21.15.66:0/1684177134 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f1454009320 con 0x7f14640f6510 2023-11-17T05:25:13.572 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:13.562+0000 7f1460ff9700 1 -- 172.21.15.66:0/1684177134 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(25 keys) v1 ==== 1074+0+0 (secure 0 0 0) 0x7f1454013f00 con 0x7f14640f6510 2023-11-17T05:25:13.572 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:13.562+0000 7f146a2fe700 1 -- 172.21.15.66:0/1684177134 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f1464106e20 con 0x7f14640f6510 2023-11-17T05:25:13.572 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:13.562+0000 7f1460ff9700 1 -- 172.21.15.66:0/1684177134 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f1454014d90 con 0x7f14640f6510 2023-11-17T05:25:13.572 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:13.562+0000 7f146a2fe700 1 -- 172.21.15.66:0/1684177134 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f14641072b0 con 0x7f14640f6510 2023-11-17T05:25:13.572 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:13.563+0000 7f1460ff9700 1 -- 172.21.15.66:0/1684177134 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 17) v1 ==== 36269+0+0 (secure 0 0 0) 0x7f14540095f0 con 0x7f14640f6510 2023-11-17T05:25:13.572 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:13.563+0000 7f146a2fe700 1 -- 172.21.15.66:0/1684177134 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f1464045ad0 con 0x7f14640f6510 2023-11-17T05:25:13.572 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:13.564+0000 7f1460ff9700 1 --2- 172.21.15.66:0/1684177134 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f144c02ffb0 0x7f144c032470 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:25:13.572 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:13.564+0000 7f1460ff9700 1 -- 172.21.15.66:0/1684177134 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(5..5 src has 1..5) v4 ==== 1270+0+0 (secure 0 0 0) 0x7f1454040f90 con 0x7f14640f6510 2023-11-17T05:25:13.572 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:13.564+0000 7f14637fe700 1 --2- 172.21.15.66:0/1684177134 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f144c02ffb0 0x7f144c032470 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:25:13.572 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:13.564+0000 7f14637fe700 1 --2- 172.21.15.66:0/1684177134 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f144c02ffb0 0x7f144c032470 secure :-1 s=READY pgs=58 cs=0 l=1 rev1=1 rx=0x7f14580096a0 tx=0x7f1458006b40).ready entity=mgr.14164 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:25:13.572 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:13.568+0000 7f1460ff9700 1 -- 172.21.15.66:0/1684177134 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+152212 (secure 0 0 0) 0x7f1454014420 con 0x7f14640f6510 2023-11-17T05:25:13.789 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:13.787+0000 7f146a2fe700 1 -- 172.21.15.66:0/1684177134 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "mon dump", "format": "json"} v 0) v1 -- 0x7f14640fbf60 con 0x7f14640f6510 2023-11-17T05:25:13.789 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:13.788+0000 7f1460ff9700 1 -- 172.21.15.66:0/1684177134 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mon_command_ack([{"prefix": "mon dump", "format": "json"}]=0 dumped monmap epoch 1 v1) v1 ==== 95+0+690 (secure 0 0 0) 0x7f145401d7f0 con 0x7f14640f6510 2023-11-17T05:25:13.789 INFO:teuthology.orchestra.run.smithi066.stdout: 2023-11-17T05:25:13.790 INFO:teuthology.orchestra.run.smithi066.stdout:{"epoch":1,"fsid":"ed6470e2-8508-11ee-8dbe-212e2dc638e7","modified":"2023-11-17T05:20:13.934038Z","created":"2023-11-17T05:20:13.934038Z","min_mon_release":16,"min_mon_release_name":"pacific","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging"],"optional":[]},"mons":[{"rank":0,"name":"smithi040","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:3300","nonce":0},{"type":"v1","addr":"172.21.15.40:6789","nonce":0}]},"addr":"172.21.15.40:6789/0","public_addr":"172.21.15.40:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2023-11-17T05:25:13.791 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:13.789+0000 7f146a2fe700 1 -- 172.21.15.66:0/1684177134 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f144c02ffb0 msgr2=0x7f144c032470 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:25:13.791 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:13.789+0000 7f146a2fe700 1 --2- 172.21.15.66:0/1684177134 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f144c02ffb0 0x7f144c032470 secure :-1 s=READY pgs=58 cs=0 l=1 rev1=1 rx=0x7f14580096a0 tx=0x7f1458006b40).stop 2023-11-17T05:25:13.791 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:13.789+0000 7f146a2fe700 1 -- 172.21.15.66:0/1684177134 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f14640f6510 msgr2=0x7f1464106730 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:25:13.791 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:13.789+0000 7f146a2fe700 1 --2- 172.21.15.66:0/1684177134 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f14640f6510 0x7f1464106730 secure :-1 s=READY pgs=186 cs=0 l=1 rev1=1 rx=0x7f1454013510 tx=0x7f145400b040).stop 2023-11-17T05:25:13.791 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:13.789+0000 7f146a2fe700 1 -- 172.21.15.66:0/1684177134 shutdown_connections 2023-11-17T05:25:13.791 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:13.789+0000 7f146a2fe700 1 --2- 172.21.15.66:0/1684177134 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f144c02ffb0 0x7f144c032470 unknown :-1 s=CLOSED pgs=58 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:25:13.791 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:13.789+0000 7f146a2fe700 1 --2- 172.21.15.66:0/1684177134 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f14640f6510 0x7f1464106730 unknown :-1 s=CLOSED pgs=186 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:25:13.791 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:13.789+0000 7f146a2fe700 1 -- 172.21.15.66:0/1684177134 >> 172.21.15.66:0/1684177134 conn(0x7f14640f1c70 msgr2=0x7f14640f2760 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:25:13.791 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:13.790+0000 7f146a2fe700 1 -- 172.21.15.66:0/1684177134 shutdown_connections 2023-11-17T05:25:13.791 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:13.790+0000 7f146a2fe700 1 -- 172.21.15.66:0/1684177134 wait complete. 2023-11-17T05:25:13.792 INFO:teuthology.orchestra.run.smithi066.stderr:dumped monmap epoch 1 2023-11-17T05:25:14.741 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:25:14 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:25:13.194721+0000 mgr.smithi040.ogcprl (mgr.14164) 122 : cluster [DBG] pgmap v115: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:25:14.741 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:25:14 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:25:13.789643+0000 mon.smithi040 (mon.0) 174 : audit [DBG] from='client.? 172.21.15.66:0/1684177134' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-11-17T05:25:15.179 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2023-11-17T05:25:15.179 DEBUG:teuthology.orchestra.run.smithi066:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.4 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ed6470e2-8508-11ee-8dbe-212e2dc638e7 -- ceph mon dump -f json 2023-11-17T05:25:16.415 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:16.405+0000 7f1e65d2e700 1 -- 172.21.15.66:0/606514452 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f1e600f62a0 msgr2=0x7f1e600f66c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:25:16.416 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:16.405+0000 7f1e65d2e700 1 --2- 172.21.15.66:0/606514452 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f1e600f62a0 0x7f1e600f66c0 secure :-1 s=READY pgs=187 cs=0 l=1 rev1=1 rx=0x7f1e54009fb0 tx=0x7f1e54009b30).stop 2023-11-17T05:25:16.416 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:16.405+0000 7f1e65d2e700 1 -- 172.21.15.66:0/606514452 shutdown_connections 2023-11-17T05:25:16.416 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:16.405+0000 7f1e65d2e700 1 --2- 172.21.15.66:0/606514452 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f1e600f62a0 0x7f1e600f66c0 unknown :-1 s=CLOSED pgs=187 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:25:16.416 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:16.405+0000 7f1e65d2e700 1 -- 172.21.15.66:0/606514452 >> 172.21.15.66:0/606514452 conn(0x7f1e600f1aa0 msgr2=0x7f1e600f3ec0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:25:16.416 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:16.405+0000 7f1e65d2e700 1 -- 172.21.15.66:0/606514452 shutdown_connections 2023-11-17T05:25:16.416 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:16.405+0000 7f1e65d2e700 1 -- 172.21.15.66:0/606514452 wait complete. 2023-11-17T05:25:16.416 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:16.405+0000 7f1e65d2e700 1 Processor -- start 2023-11-17T05:25:16.416 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:16.405+0000 7f1e65d2e700 1 -- start start 2023-11-17T05:25:16.416 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:16.406+0000 7f1e65d2e700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f1e600f62a0 0x7f1e6006b9a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:25:16.416 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:16.406+0000 7f1e65d2e700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f1e60069e20 con 0x7f1e600f62a0 2023-11-17T05:25:16.416 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:16.406+0000 7f1e64d2c700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f1e600f62a0 0x7f1e6006b9a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:25:16.416 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:16.406+0000 7f1e64d2c700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f1e600f62a0 0x7f1e6006b9a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.66:40410/0 (socket says 172.21.15.66:40410) 2023-11-17T05:25:16.417 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:16.406+0000 7f1e64d2c700 1 -- 172.21.15.66:0/2879624188 learned_addr learned my addr 172.21.15.66:0/2879624188 (peer_addr_for_me v2:172.21.15.66:0/0) 2023-11-17T05:25:16.417 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:16.406+0000 7f1e64d2c700 1 -- 172.21.15.66:0/2879624188 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f1e54005040 con 0x7f1e600f62a0 2023-11-17T05:25:16.417 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:16.406+0000 7f1e64d2c700 1 --2- 172.21.15.66:0/2879624188 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f1e600f62a0 0x7f1e6006b9a0 secure :-1 s=READY pgs=188 cs=0 l=1 rev1=1 rx=0x7f1e54013510 tx=0x7f1e5400b040).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:25:16.417 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:16.406+0000 7f1e5d7fa700 1 -- 172.21.15.66:0/2879624188 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f1e54009320 con 0x7f1e600f62a0 2023-11-17T05:25:16.417 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:16.406+0000 7f1e65d2e700 1 -- 172.21.15.66:0/2879624188 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f1e60069fc0 con 0x7f1e600f62a0 2023-11-17T05:25:16.417 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:16.406+0000 7f1e5d7fa700 1 -- 172.21.15.66:0/2879624188 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(25 keys) v1 ==== 1074+0+0 (secure 0 0 0) 0x7f1e54013f00 con 0x7f1e600f62a0 2023-11-17T05:25:16.417 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:16.406+0000 7f1e5d7fa700 1 -- 172.21.15.66:0/2879624188 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f1e54014dc0 con 0x7f1e600f62a0 2023-11-17T05:25:16.417 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:16.407+0000 7f1e65d2e700 1 -- 172.21.15.66:0/2879624188 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f1e6006a450 con 0x7f1e600f62a0 2023-11-17T05:25:16.417 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:16.407+0000 7f1e5d7fa700 1 -- 172.21.15.66:0/2879624188 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 17) v1 ==== 36269+0+0 (secure 0 0 0) 0x7f1e540095f0 con 0x7f1e600f62a0 2023-11-17T05:25:16.417 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:16.407+0000 7f1e65d2e700 1 -- 172.21.15.66:0/2879624188 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f1e60059100 con 0x7f1e600f62a0 2023-11-17T05:25:16.417 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:16.407+0000 7f1e5d7fa700 1 --2- 172.21.15.66:0/2879624188 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f1e5002ff60 0x7f1e50032420 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:25:16.417 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:16.408+0000 7f1e5d7fa700 1 -- 172.21.15.66:0/2879624188 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(5..5 src has 1..5) v4 ==== 1270+0+0 (secure 0 0 0) 0x7f1e54041080 con 0x7f1e600f62a0 2023-11-17T05:25:16.417 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:16.408+0000 7f1e5ffff700 1 --2- 172.21.15.66:0/2879624188 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f1e5002ff60 0x7f1e50032420 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:25:16.417 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:16.408+0000 7f1e5ffff700 1 --2- 172.21.15.66:0/2879624188 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f1e5002ff60 0x7f1e50032420 secure :-1 s=READY pgs=59 cs=0 l=1 rev1=1 rx=0x7f1e4c0096a0 tx=0x7f1e4c006b40).ready entity=mgr.14164 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:25:16.419 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:16.412+0000 7f1e5d7fa700 1 -- 172.21.15.66:0/2879624188 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+152212 (secure 0 0 0) 0x7f1e5401c790 con 0x7f1e600f62a0 2023-11-17T05:25:16.630 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:16.629+0000 7f1e65d2e700 1 -- 172.21.15.66:0/2879624188 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "mon dump", "format": "json"} v 0) v1 -- 0x7f1e600fbce0 con 0x7f1e600f62a0 2023-11-17T05:25:16.631 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:16.629+0000 7f1e5d7fa700 1 -- 172.21.15.66:0/2879624188 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mon_command_ack([{"prefix": "mon dump", "format": "json"}]=0 dumped monmap epoch 1 v1) v1 ==== 95+0+690 (secure 0 0 0) 0x7f1e54010230 con 0x7f1e600f62a0 2023-11-17T05:25:16.631 INFO:teuthology.orchestra.run.smithi066.stdout: 2023-11-17T05:25:16.631 INFO:teuthology.orchestra.run.smithi066.stdout:{"epoch":1,"fsid":"ed6470e2-8508-11ee-8dbe-212e2dc638e7","modified":"2023-11-17T05:20:13.934038Z","created":"2023-11-17T05:20:13.934038Z","min_mon_release":16,"min_mon_release_name":"pacific","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging"],"optional":[]},"mons":[{"rank":0,"name":"smithi040","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:3300","nonce":0},{"type":"v1","addr":"172.21.15.40:6789","nonce":0}]},"addr":"172.21.15.40:6789/0","public_addr":"172.21.15.40:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2023-11-17T05:25:16.632 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:16.631+0000 7f1e65d2e700 1 -- 172.21.15.66:0/2879624188 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f1e5002ff60 msgr2=0x7f1e50032420 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:25:16.632 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:16.631+0000 7f1e65d2e700 1 --2- 172.21.15.66:0/2879624188 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f1e5002ff60 0x7f1e50032420 secure :-1 s=READY pgs=59 cs=0 l=1 rev1=1 rx=0x7f1e4c0096a0 tx=0x7f1e4c006b40).stop 2023-11-17T05:25:16.633 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:16.631+0000 7f1e65d2e700 1 -- 172.21.15.66:0/2879624188 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f1e600f62a0 msgr2=0x7f1e6006b9a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:25:16.633 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:16.631+0000 7f1e65d2e700 1 --2- 172.21.15.66:0/2879624188 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f1e600f62a0 0x7f1e6006b9a0 secure :-1 s=READY pgs=188 cs=0 l=1 rev1=1 rx=0x7f1e54013510 tx=0x7f1e5400b040).stop 2023-11-17T05:25:16.633 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:16.631+0000 7f1e65d2e700 1 -- 172.21.15.66:0/2879624188 shutdown_connections 2023-11-17T05:25:16.633 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:16.631+0000 7f1e65d2e700 1 --2- 172.21.15.66:0/2879624188 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f1e5002ff60 0x7f1e50032420 unknown :-1 s=CLOSED pgs=59 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:25:16.633 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:16.631+0000 7f1e65d2e700 1 --2- 172.21.15.66:0/2879624188 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f1e600f62a0 0x7f1e6006b9a0 unknown :-1 s=CLOSED pgs=188 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:25:16.633 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:16.631+0000 7f1e65d2e700 1 -- 172.21.15.66:0/2879624188 >> 172.21.15.66:0/2879624188 conn(0x7f1e600f1aa0 msgr2=0x7f1e600f2510 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:25:16.633 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:16.631+0000 7f1e65d2e700 1 -- 172.21.15.66:0/2879624188 shutdown_connections 2023-11-17T05:25:16.633 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:16.631+0000 7f1e65d2e700 1 -- 172.21.15.66:0/2879624188 wait complete. 2023-11-17T05:25:16.633 INFO:teuthology.orchestra.run.smithi066.stderr:dumped monmap epoch 1 2023-11-17T05:25:16.739 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:25:16 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:25:15.194958+0000 mgr.smithi040.ogcprl (mgr.14164) 123 : cluster [DBG] pgmap v116: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:25:17.739 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:25:17 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:25:16.631116+0000 mon.smithi040 (mon.0) 175 : audit [DBG] from='client.? 172.21.15.66:0/2879624188' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-11-17T05:25:18.010 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2023-11-17T05:25:18.011 DEBUG:teuthology.orchestra.run.smithi066:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.4 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ed6470e2-8508-11ee-8dbe-212e2dc638e7 -- ceph mon dump -f json 2023-11-17T05:25:18.739 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:25:18 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:25:17.195283+0000 mgr.smithi040.ogcprl (mgr.14164) 124 : cluster [DBG] pgmap v117: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:25:19.203 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:19.199+0000 7f4280456700 1 -- 172.21.15.66:0/1055492198 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f42780601e0 msgr2=0x7f42780f9190 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:25:19.203 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:19.199+0000 7f4280456700 1 --2- 172.21.15.66:0/1055492198 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f42780601e0 0x7f42780f9190 secure :-1 s=READY pgs=189 cs=0 l=1 rev1=1 rx=0x7f4274009fb0 tx=0x7f4274009b30).stop 2023-11-17T05:25:19.203 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:19.200+0000 7f4280456700 1 -- 172.21.15.66:0/1055492198 shutdown_connections 2023-11-17T05:25:19.203 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:19.200+0000 7f4280456700 1 --2- 172.21.15.66:0/1055492198 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f42780601e0 0x7f42780f9190 unknown :-1 s=CLOSED pgs=189 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:25:19.203 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:19.200+0000 7f4280456700 1 -- 172.21.15.66:0/1055492198 >> 172.21.15.66:0/1055492198 conn(0x7f42780f0b70 msgr2=0x7f42780f2f90 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:25:19.203 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:19.200+0000 7f4280456700 1 -- 172.21.15.66:0/1055492198 shutdown_connections 2023-11-17T05:25:19.203 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:19.200+0000 7f4280456700 1 -- 172.21.15.66:0/1055492198 wait complete. 2023-11-17T05:25:19.203 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:19.200+0000 7f4280456700 1 Processor -- start 2023-11-17T05:25:19.203 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:19.200+0000 7f4280456700 1 -- start start 2023-11-17T05:25:19.204 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:19.200+0000 7f4280456700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f42780601e0 0x7f42780f6c10 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:25:19.204 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:19.200+0000 7f4280456700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f42780f5090 con 0x7f42780601e0 2023-11-17T05:25:19.204 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:19.201+0000 7f427e1f2700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f42780601e0 0x7f42780f6c10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:25:19.204 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:19.201+0000 7f427e1f2700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f42780601e0 0x7f42780f6c10 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.66:40434/0 (socket says 172.21.15.66:40434) 2023-11-17T05:25:19.204 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:19.201+0000 7f427e1f2700 1 -- 172.21.15.66:0/69740098 learned_addr learned my addr 172.21.15.66:0/69740098 (peer_addr_for_me v2:172.21.15.66:0/0) 2023-11-17T05:25:19.204 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:19.201+0000 7f427e1f2700 1 -- 172.21.15.66:0/69740098 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f4274005040 con 0x7f42780601e0 2023-11-17T05:25:19.204 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:19.201+0000 7f427e1f2700 1 --2- 172.21.15.66:0/69740098 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f42780601e0 0x7f42780f6c10 secure :-1 s=READY pgs=190 cs=0 l=1 rev1=1 rx=0x7f4274004dd0 tx=0x7f4274009b30).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:25:19.204 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:19.201+0000 7f426effd700 1 -- 172.21.15.66:0/69740098 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f4274018070 con 0x7f42780601e0 2023-11-17T05:25:19.205 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:19.201+0000 7f426effd700 1 -- 172.21.15.66:0/69740098 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(25 keys) v1 ==== 1074+0+0 (secure 0 0 0) 0x7f4274013410 con 0x7f42780601e0 2023-11-17T05:25:19.205 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:19.201+0000 7f4280456700 1 -- 172.21.15.66:0/69740098 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f42780f5230 con 0x7f42780601e0 2023-11-17T05:25:19.205 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:19.201+0000 7f426effd700 1 -- 172.21.15.66:0/69740098 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f427401c3c0 con 0x7f42780601e0 2023-11-17T05:25:19.205 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:19.201+0000 7f4280456700 1 -- 172.21.15.66:0/69740098 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f42780f56c0 con 0x7f42780601e0 2023-11-17T05:25:19.206 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:19.202+0000 7f426effd700 1 -- 172.21.15.66:0/69740098 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 17) v1 ==== 36269+0+0 (secure 0 0 0) 0x7f427401d420 con 0x7f42780601e0 2023-11-17T05:25:19.206 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:19.202+0000 7f4280456700 1 -- 172.21.15.66:0/69740098 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f4278059100 con 0x7f42780601e0 2023-11-17T05:25:19.206 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:19.202+0000 7f426effd700 1 --2- 172.21.15.66:0/69740098 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f426402ff60 0x7f4264032420 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:25:19.206 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:19.202+0000 7f426effd700 1 -- 172.21.15.66:0/69740098 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(5..5 src has 1..5) v4 ==== 1270+0+0 (secure 0 0 0) 0x7f42740413f0 con 0x7f42780601e0 2023-11-17T05:25:19.206 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:19.203+0000 7f427d9f1700 1 --2- 172.21.15.66:0/69740098 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f426402ff60 0x7f4264032420 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:25:19.206 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:19.203+0000 7f427d9f1700 1 --2- 172.21.15.66:0/69740098 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f426402ff60 0x7f4264032420 secure :-1 s=READY pgs=60 cs=0 l=1 rev1=1 rx=0x7f42680096a0 tx=0x7f4268006b40).ready entity=mgr.14164 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:25:19.209 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:19.207+0000 7f426effd700 1 -- 172.21.15.66:0/69740098 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+152212 (secure 0 0 0) 0x7f427401d820 con 0x7f42780601e0 2023-11-17T05:25:19.429 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:19.428+0000 7f4280456700 1 -- 172.21.15.66:0/69740098 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "mon dump", "format": "json"} v 0) v1 -- 0x7f42780fbf60 con 0x7f42780601e0 2023-11-17T05:25:19.431 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:19.430+0000 7f426effd700 1 -- 172.21.15.66:0/69740098 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mon_command_ack([{"prefix": "mon dump", "format": "json"}]=0 dumped monmap epoch 1 v1) v1 ==== 95+0+690 (secure 0 0 0) 0x7f427401b370 con 0x7f42780601e0 2023-11-17T05:25:19.432 INFO:teuthology.orchestra.run.smithi066.stdout: 2023-11-17T05:25:19.432 INFO:teuthology.orchestra.run.smithi066.stdout:{"epoch":1,"fsid":"ed6470e2-8508-11ee-8dbe-212e2dc638e7","modified":"2023-11-17T05:20:13.934038Z","created":"2023-11-17T05:20:13.934038Z","min_mon_release":16,"min_mon_release_name":"pacific","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging"],"optional":[]},"mons":[{"rank":0,"name":"smithi040","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:3300","nonce":0},{"type":"v1","addr":"172.21.15.40:6789","nonce":0}]},"addr":"172.21.15.40:6789/0","public_addr":"172.21.15.40:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2023-11-17T05:25:19.441 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:19.432+0000 7f4280456700 1 -- 172.21.15.66:0/69740098 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f426402ff60 msgr2=0x7f4264032420 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:25:19.441 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:19.432+0000 7f4280456700 1 --2- 172.21.15.66:0/69740098 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f426402ff60 0x7f4264032420 secure :-1 s=READY pgs=60 cs=0 l=1 rev1=1 rx=0x7f42680096a0 tx=0x7f4268006b40).stop 2023-11-17T05:25:19.441 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:19.432+0000 7f4280456700 1 -- 172.21.15.66:0/69740098 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f42780601e0 msgr2=0x7f42780f6c10 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:25:19.456 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:19.432+0000 7f4280456700 1 --2- 172.21.15.66:0/69740098 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f42780601e0 0x7f42780f6c10 secure :-1 s=READY pgs=190 cs=0 l=1 rev1=1 rx=0x7f4274004dd0 tx=0x7f4274009b30).stop 2023-11-17T05:25:19.456 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:19.432+0000 7f4280456700 1 -- 172.21.15.66:0/69740098 shutdown_connections 2023-11-17T05:25:19.456 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:19.432+0000 7f4280456700 1 --2- 172.21.15.66:0/69740098 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f426402ff60 0x7f4264032420 unknown :-1 s=CLOSED pgs=60 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:25:19.456 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:19.432+0000 7f4280456700 1 --2- 172.21.15.66:0/69740098 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f42780601e0 0x7f42780f6c10 unknown :-1 s=CLOSED pgs=190 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:25:19.456 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:19.432+0000 7f4280456700 1 -- 172.21.15.66:0/69740098 >> 172.21.15.66:0/69740098 conn(0x7f42780f0b70 msgr2=0x7f42780f1670 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:25:19.456 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:19.432+0000 7f4280456700 1 -- 172.21.15.66:0/69740098 shutdown_connections 2023-11-17T05:25:19.456 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:19.432+0000 7f4280456700 1 -- 172.21.15.66:0/69740098 wait complete. 2023-11-17T05:25:19.456 INFO:teuthology.orchestra.run.smithi066.stderr:dumped monmap epoch 1 2023-11-17T05:25:20.738 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:25:20 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:25:19.195638+0000 mgr.smithi040.ogcprl (mgr.14164) 125 : cluster [DBG] pgmap v118: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:25:20.739 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:25:20 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:25:19.431832+0000 mon.smithi040 (mon.0) 176 : audit [DBG] from='client.? 172.21.15.66:0/69740098' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-11-17T05:25:20.815 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2023-11-17T05:25:20.816 DEBUG:teuthology.orchestra.run.smithi066:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.4 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ed6470e2-8508-11ee-8dbe-212e2dc638e7 -- ceph mon dump -f json 2023-11-17T05:25:22.051 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:22.049+0000 7fb38d7c8700 1 -- 172.21.15.66:0/2975115709 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fb3880f47c0 msgr2=0x7fb3880f4be0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:25:22.051 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:22.049+0000 7fb38d7c8700 1 --2- 172.21.15.66:0/2975115709 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fb3880f47c0 0x7fb3880f4be0 secure :-1 s=READY pgs=191 cs=0 l=1 rev1=1 rx=0x7fb378009fb0 tx=0x7fb378009b30).stop 2023-11-17T05:25:22.051 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:22.049+0000 7fb38d7c8700 1 -- 172.21.15.66:0/2975115709 shutdown_connections 2023-11-17T05:25:22.051 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:22.049+0000 7fb38d7c8700 1 --2- 172.21.15.66:0/2975115709 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fb3880f47c0 0x7fb3880f4be0 unknown :-1 s=CLOSED pgs=191 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:25:22.051 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:22.049+0000 7fb38d7c8700 1 -- 172.21.15.66:0/2975115709 >> 172.21.15.66:0/2975115709 conn(0x7fb3880f02e0 msgr2=0x7fb3880f2700 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:25:22.051 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:22.049+0000 7fb38d7c8700 1 -- 172.21.15.66:0/2975115709 shutdown_connections 2023-11-17T05:25:22.051 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:22.049+0000 7fb38d7c8700 1 -- 172.21.15.66:0/2975115709 wait complete. 2023-11-17T05:25:22.053 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:22.050+0000 7fb38d7c8700 1 Processor -- start 2023-11-17T05:25:22.053 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:22.050+0000 7fb38d7c8700 1 -- start start 2023-11-17T05:25:22.053 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:22.050+0000 7fb38d7c8700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fb3880f47c0 0x7fb388106700 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:25:22.053 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:22.050+0000 7fb38d7c8700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fb388106bf0 con 0x7fb3880f47c0 2023-11-17T05:25:22.053 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:22.051+0000 7fb386ffd700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fb3880f47c0 0x7fb388106700 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:25:22.053 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:22.051+0000 7fb386ffd700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fb3880f47c0 0x7fb388106700 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.66:40452/0 (socket says 172.21.15.66:40452) 2023-11-17T05:25:22.053 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:22.051+0000 7fb386ffd700 1 -- 172.21.15.66:0/1287703681 learned_addr learned my addr 172.21.15.66:0/1287703681 (peer_addr_for_me v2:172.21.15.66:0/0) 2023-11-17T05:25:22.053 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:22.051+0000 7fb386ffd700 1 -- 172.21.15.66:0/1287703681 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fb378005040 con 0x7fb3880f47c0 2023-11-17T05:25:22.053 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:22.051+0000 7fb386ffd700 1 --2- 172.21.15.66:0/1287703681 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fb3880f47c0 0x7fb388106700 secure :-1 s=READY pgs=192 cs=0 l=1 rev1=1 rx=0x7fb378013510 tx=0x7fb37800b040).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:25:22.053 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:22.051+0000 7fb37f7fe700 1 -- 172.21.15.66:0/1287703681 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7fb378009320 con 0x7fb3880f47c0 2023-11-17T05:25:22.053 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:22.051+0000 7fb37f7fe700 1 -- 172.21.15.66:0/1287703681 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(25 keys) v1 ==== 1074+0+0 (secure 0 0 0) 0x7fb378013f00 con 0x7fb3880f47c0 2023-11-17T05:25:22.054 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:22.051+0000 7fb38d7c8700 1 -- 172.21.15.66:0/1287703681 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fb388106df0 con 0x7fb3880f47c0 2023-11-17T05:25:22.054 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:22.051+0000 7fb37f7fe700 1 -- 172.21.15.66:0/1287703681 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7fb378014d90 con 0x7fb3880f47c0 2023-11-17T05:25:22.054 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:22.051+0000 7fb38d7c8700 1 -- 172.21.15.66:0/1287703681 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fb388107280 con 0x7fb3880f47c0 2023-11-17T05:25:22.054 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:22.052+0000 7fb37f7fe700 1 -- 172.21.15.66:0/1287703681 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 17) v1 ==== 36269+0+0 (secure 0 0 0) 0x7fb3780095f0 con 0x7fb3880f47c0 2023-11-17T05:25:22.055 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:22.052+0000 7fb38d7c8700 1 -- 172.21.15.66:0/1287703681 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fb388045ad0 con 0x7fb3880f47c0 2023-11-17T05:25:22.055 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:22.053+0000 7fb37f7fe700 1 --2- 172.21.15.66:0/1287703681 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fb37402ffb0 0x7fb374032470 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:25:22.055 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:22.053+0000 7fb37f7fe700 1 -- 172.21.15.66:0/1287703681 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(5..5 src has 1..5) v4 ==== 1270+0+0 (secure 0 0 0) 0x7fb378040f90 con 0x7fb3880f47c0 2023-11-17T05:25:22.055 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:22.053+0000 7fb3867fc700 1 --2- 172.21.15.66:0/1287703681 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fb37402ffb0 0x7fb374032470 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:25:22.055 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:22.053+0000 7fb3867fc700 1 --2- 172.21.15.66:0/1287703681 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fb37402ffb0 0x7fb374032470 secure :-1 s=READY pgs=61 cs=0 l=1 rev1=1 rx=0x7fb3700096a0 tx=0x7fb370006b40).ready entity=mgr.14164 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:25:22.059 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:22.058+0000 7fb37f7fe700 1 -- 172.21.15.66:0/1287703681 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+152212 (secure 0 0 0) 0x7fb378014420 con 0x7fb3880f47c0 2023-11-17T05:25:22.276 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:22.275+0000 7fb38d7c8700 1 -- 172.21.15.66:0/1287703681 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "mon dump", "format": "json"} v 0) v1 -- 0x7fb388108130 con 0x7fb3880f47c0 2023-11-17T05:25:22.277 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:22.275+0000 7fb37f7fe700 1 -- 172.21.15.66:0/1287703681 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mon_command_ack([{"prefix": "mon dump", "format": "json"}]=0 dumped monmap epoch 1 v1) v1 ==== 95+0+690 (secure 0 0 0) 0x7fb37801d7f0 con 0x7fb3880f47c0 2023-11-17T05:25:22.277 INFO:teuthology.orchestra.run.smithi066.stdout: 2023-11-17T05:25:22.277 INFO:teuthology.orchestra.run.smithi066.stdout:{"epoch":1,"fsid":"ed6470e2-8508-11ee-8dbe-212e2dc638e7","modified":"2023-11-17T05:20:13.934038Z","created":"2023-11-17T05:20:13.934038Z","min_mon_release":16,"min_mon_release_name":"pacific","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging"],"optional":[]},"mons":[{"rank":0,"name":"smithi040","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:3300","nonce":0},{"type":"v1","addr":"172.21.15.40:6789","nonce":0}]},"addr":"172.21.15.40:6789/0","public_addr":"172.21.15.40:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2023-11-17T05:25:22.278 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:22.277+0000 7fb38d7c8700 1 -- 172.21.15.66:0/1287703681 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fb37402ffb0 msgr2=0x7fb374032470 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:25:22.278 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:22.277+0000 7fb38d7c8700 1 --2- 172.21.15.66:0/1287703681 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fb37402ffb0 0x7fb374032470 secure :-1 s=READY pgs=61 cs=0 l=1 rev1=1 rx=0x7fb3700096a0 tx=0x7fb370006b40).stop 2023-11-17T05:25:22.278 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:22.277+0000 7fb38d7c8700 1 -- 172.21.15.66:0/1287703681 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fb3880f47c0 msgr2=0x7fb388106700 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:25:22.278 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:22.277+0000 7fb38d7c8700 1 --2- 172.21.15.66:0/1287703681 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fb3880f47c0 0x7fb388106700 secure :-1 s=READY pgs=192 cs=0 l=1 rev1=1 rx=0x7fb378013510 tx=0x7fb37800b040).stop 2023-11-17T05:25:22.279 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:22.277+0000 7fb38d7c8700 1 -- 172.21.15.66:0/1287703681 shutdown_connections 2023-11-17T05:25:22.279 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:22.277+0000 7fb38d7c8700 1 --2- 172.21.15.66:0/1287703681 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fb37402ffb0 0x7fb374032470 unknown :-1 s=CLOSED pgs=61 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:25:22.279 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:22.277+0000 7fb38d7c8700 1 --2- 172.21.15.66:0/1287703681 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fb3880f47c0 0x7fb388106700 unknown :-1 s=CLOSED pgs=192 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:25:22.279 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:22.277+0000 7fb38d7c8700 1 -- 172.21.15.66:0/1287703681 >> 172.21.15.66:0/1287703681 conn(0x7fb3880f02e0 msgr2=0x7fb3880f0db0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:25:22.279 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:22.277+0000 7fb38d7c8700 1 -- 172.21.15.66:0/1287703681 shutdown_connections 2023-11-17T05:25:22.279 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:22.277+0000 7fb38d7c8700 1 -- 172.21.15.66:0/1287703681 wait complete. 2023-11-17T05:25:22.279 INFO:teuthology.orchestra.run.smithi066.stderr:dumped monmap epoch 1 2023-11-17T05:25:22.739 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:25:22 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:25:21.195910+0000 mgr.smithi040.ogcprl (mgr.14164) 126 : cluster [DBG] pgmap v119: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:25:22.739 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:25:22 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:25:22.277058+0000 mon.smithi040 (mon.0) 177 : audit [DBG] from='client.? 172.21.15.66:0/1287703681' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-11-17T05:25:23.701 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2023-11-17T05:25:23.701 DEBUG:teuthology.orchestra.run.smithi066:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.4 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ed6470e2-8508-11ee-8dbe-212e2dc638e7 -- ceph mon dump -f json 2023-11-17T05:25:24.739 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:25:24 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:25:23.196247+0000 mgr.smithi040.ogcprl (mgr.14164) 127 : cluster [DBG] pgmap v120: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:25:24.855 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:24.854+0000 7f824586c700 1 -- 172.21.15.66:0/3915725989 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f82400f62a0 msgr2=0x7f82400f66c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:25:24.856 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:24.854+0000 7f824586c700 1 --2- 172.21.15.66:0/3915725989 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f82400f62a0 0x7f82400f66c0 secure :-1 s=READY pgs=193 cs=0 l=1 rev1=1 rx=0x7f8234009fb0 tx=0x7f8234009b30).stop 2023-11-17T05:25:24.856 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:24.854+0000 7f824586c700 1 -- 172.21.15.66:0/3915725989 shutdown_connections 2023-11-17T05:25:24.856 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:24.854+0000 7f824586c700 1 --2- 172.21.15.66:0/3915725989 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f82400f62a0 0x7f82400f66c0 unknown :-1 s=CLOSED pgs=193 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:25:24.856 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:24.854+0000 7f824586c700 1 -- 172.21.15.66:0/3915725989 >> 172.21.15.66:0/3915725989 conn(0x7f82400f1aa0 msgr2=0x7f82400f3ec0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:25:24.856 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:24.854+0000 7f824586c700 1 -- 172.21.15.66:0/3915725989 shutdown_connections 2023-11-17T05:25:24.856 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:24.854+0000 7f824586c700 1 -- 172.21.15.66:0/3915725989 wait complete. 2023-11-17T05:25:24.856 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:24.854+0000 7f824586c700 1 Processor -- start 2023-11-17T05:25:24.856 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:24.854+0000 7f824586c700 1 -- start start 2023-11-17T05:25:24.857 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:24.855+0000 7f824586c700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f82400f62a0 0x7f824006b9a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:25:24.857 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:24.855+0000 7f824586c700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f8240069e20 con 0x7f82400f62a0 2023-11-17T05:25:24.857 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:24.855+0000 7f824486a700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f82400f62a0 0x7f824006b9a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:25:24.858 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:24.855+0000 7f824486a700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f82400f62a0 0x7f824006b9a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.66:42018/0 (socket says 172.21.15.66:42018) 2023-11-17T05:25:24.858 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:24.855+0000 7f824486a700 1 -- 172.21.15.66:0/52103137 learned_addr learned my addr 172.21.15.66:0/52103137 (peer_addr_for_me v2:172.21.15.66:0/0) 2023-11-17T05:25:24.858 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:24.855+0000 7f824486a700 1 -- 172.21.15.66:0/52103137 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f8234005040 con 0x7f82400f62a0 2023-11-17T05:25:24.858 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:24.855+0000 7f824486a700 1 --2- 172.21.15.66:0/52103137 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f82400f62a0 0x7f824006b9a0 secure :-1 s=READY pgs=194 cs=0 l=1 rev1=1 rx=0x7f8234013510 tx=0x7f823400b040).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:25:24.858 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:24.855+0000 7f823d7fa700 1 -- 172.21.15.66:0/52103137 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f8234009320 con 0x7f82400f62a0 2023-11-17T05:25:24.858 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:24.855+0000 7f824586c700 1 -- 172.21.15.66:0/52103137 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f8240069fc0 con 0x7f82400f62a0 2023-11-17T05:25:24.858 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:24.855+0000 7f823d7fa700 1 -- 172.21.15.66:0/52103137 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(25 keys) v1 ==== 1074+0+0 (secure 0 0 0) 0x7f8234013f00 con 0x7f82400f62a0 2023-11-17T05:25:24.858 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:24.856+0000 7f823d7fa700 1 -- 172.21.15.66:0/52103137 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f8234014dc0 con 0x7f82400f62a0 2023-11-17T05:25:24.858 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:24.856+0000 7f824586c700 1 -- 172.21.15.66:0/52103137 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f824006a450 con 0x7f82400f62a0 2023-11-17T05:25:24.858 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:24.856+0000 7f823d7fa700 1 -- 172.21.15.66:0/52103137 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 17) v1 ==== 36269+0+0 (secure 0 0 0) 0x7f82340095f0 con 0x7f82400f62a0 2023-11-17T05:25:24.859 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:24.856+0000 7f824586c700 1 -- 172.21.15.66:0/52103137 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f8240059100 con 0x7f82400f62a0 2023-11-17T05:25:24.859 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:24.856+0000 7f823d7fa700 1 --2- 172.21.15.66:0/52103137 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f823002ff60 0x7f8230032420 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:25:24.859 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:24.857+0000 7f823d7fa700 1 -- 172.21.15.66:0/52103137 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(5..5 src has 1..5) v4 ==== 1270+0+0 (secure 0 0 0) 0x7f82340410c0 con 0x7f82400f62a0 2023-11-17T05:25:24.859 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:24.857+0000 7f823ffff700 1 --2- 172.21.15.66:0/52103137 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f823002ff60 0x7f8230032420 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:25:24.859 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:24.857+0000 7f823ffff700 1 --2- 172.21.15.66:0/52103137 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f823002ff60 0x7f8230032420 secure :-1 s=READY pgs=62 cs=0 l=1 rev1=1 rx=0x7f822c0096a0 tx=0x7f822c006b40).ready entity=mgr.14164 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:25:24.863 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:24.862+0000 7f823d7fa700 1 -- 172.21.15.66:0/52103137 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+152212 (secure 0 0 0) 0x7f8234014420 con 0x7f82400f62a0 2023-11-17T05:25:25.084 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:25.083+0000 7f824586c700 1 -- 172.21.15.66:0/52103137 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "mon dump", "format": "json"} v 0) v1 -- 0x7f82400fbce0 con 0x7f82400f62a0 2023-11-17T05:25:25.131 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:25.083+0000 7f823d7fa700 1 -- 172.21.15.66:0/52103137 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mon_command_ack([{"prefix": "mon dump", "format": "json"}]=0 dumped monmap epoch 1 v1) v1 ==== 95+0+690 (secure 0 0 0) 0x7f8234010230 con 0x7f82400f62a0 2023-11-17T05:25:25.131 INFO:teuthology.orchestra.run.smithi066.stdout: 2023-11-17T05:25:25.131 INFO:teuthology.orchestra.run.smithi066.stdout:{"epoch":1,"fsid":"ed6470e2-8508-11ee-8dbe-212e2dc638e7","modified":"2023-11-17T05:20:13.934038Z","created":"2023-11-17T05:20:13.934038Z","min_mon_release":16,"min_mon_release_name":"pacific","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging"],"optional":[]},"mons":[{"rank":0,"name":"smithi040","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:3300","nonce":0},{"type":"v1","addr":"172.21.15.40:6789","nonce":0}]},"addr":"172.21.15.40:6789/0","public_addr":"172.21.15.40:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2023-11-17T05:25:25.132 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:25.085+0000 7f824586c700 1 -- 172.21.15.66:0/52103137 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f823002ff60 msgr2=0x7f8230032420 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:25:25.132 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:25.085+0000 7f824586c700 1 --2- 172.21.15.66:0/52103137 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f823002ff60 0x7f8230032420 secure :-1 s=READY pgs=62 cs=0 l=1 rev1=1 rx=0x7f822c0096a0 tx=0x7f822c006b40).stop 2023-11-17T05:25:25.132 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:25.085+0000 7f824586c700 1 -- 172.21.15.66:0/52103137 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f82400f62a0 msgr2=0x7f824006b9a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:25:25.132 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:25.085+0000 7f824586c700 1 --2- 172.21.15.66:0/52103137 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f82400f62a0 0x7f824006b9a0 secure :-1 s=READY pgs=194 cs=0 l=1 rev1=1 rx=0x7f8234013510 tx=0x7f823400b040).stop 2023-11-17T05:25:25.132 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:25.085+0000 7f824586c700 1 -- 172.21.15.66:0/52103137 shutdown_connections 2023-11-17T05:25:25.133 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:25.085+0000 7f824586c700 1 --2- 172.21.15.66:0/52103137 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f823002ff60 0x7f8230032420 unknown :-1 s=CLOSED pgs=62 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:25:25.133 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:25.085+0000 7f824586c700 1 --2- 172.21.15.66:0/52103137 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f82400f62a0 0x7f824006b9a0 unknown :-1 s=CLOSED pgs=194 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:25:25.133 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:25.085+0000 7f824586c700 1 -- 172.21.15.66:0/52103137 >> 172.21.15.66:0/52103137 conn(0x7f82400f1aa0 msgr2=0x7f82400f2510 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:25:25.133 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:25.085+0000 7f824586c700 1 -- 172.21.15.66:0/52103137 shutdown_connections 2023-11-17T05:25:25.133 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:25.085+0000 7f824586c700 1 -- 172.21.15.66:0/52103137 wait complete. 2023-11-17T05:25:25.133 INFO:teuthology.orchestra.run.smithi066.stderr:dumped monmap epoch 1 2023-11-17T05:25:25.739 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:25:25 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:25:25.085091+0000 mon.smithi040 (mon.0) 178 : audit [DBG] from='client.? 172.21.15.66:0/52103137' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-11-17T05:25:26.464 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2023-11-17T05:25:26.481 DEBUG:teuthology.orchestra.run.smithi066:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.4 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ed6470e2-8508-11ee-8dbe-212e2dc638e7 -- ceph mon dump -f json 2023-11-17T05:25:26.739 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:25:26 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:25:25.196571+0000 mgr.smithi040.ogcprl (mgr.14164) 128 : cluster [DBG] pgmap v121: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:25:27.680 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:27.678+0000 7fdce07f8700 1 -- 172.21.15.66:0/1933993974 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fdcd80f5410 msgr2=0x7fdcd80f5830 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:25:27.680 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:27.678+0000 7fdce07f8700 1 --2- 172.21.15.66:0/1933993974 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fdcd80f5410 0x7fdcd80f5830 secure :-1 s=READY pgs=195 cs=0 l=1 rev1=1 rx=0x7fdcc8009fb0 tx=0x7fdcc8009b30).stop 2023-11-17T05:25:27.680 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:27.678+0000 7fdce07f8700 1 -- 172.21.15.66:0/1933993974 shutdown_connections 2023-11-17T05:25:27.680 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:27.678+0000 7fdce07f8700 1 --2- 172.21.15.66:0/1933993974 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fdcd80f5410 0x7fdcd80f5830 unknown :-1 s=CLOSED pgs=195 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:25:27.680 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:27.678+0000 7fdce07f8700 1 -- 172.21.15.66:0/1933993974 >> 172.21.15.66:0/1933993974 conn(0x7fdcd80f0b70 msgr2=0x7fdcd80f2f90 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:25:27.680 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:27.678+0000 7fdce07f8700 1 -- 172.21.15.66:0/1933993974 shutdown_connections 2023-11-17T05:25:27.680 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:27.678+0000 7fdce07f8700 1 -- 172.21.15.66:0/1933993974 wait complete. 2023-11-17T05:25:27.682 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:27.679+0000 7fdce07f8700 1 Processor -- start 2023-11-17T05:25:27.682 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:27.679+0000 7fdce07f8700 1 -- start start 2023-11-17T05:25:27.682 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:27.679+0000 7fdce07f8700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fdcd80f5410 0x7fdcd8106720 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:25:27.682 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:27.679+0000 7fdce07f8700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fdcd8106c10 con 0x7fdcd80f5410 2023-11-17T05:25:27.683 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:27.679+0000 7fdcde594700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fdcd80f5410 0x7fdcd8106720 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:25:27.683 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:27.679+0000 7fdcde594700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fdcd80f5410 0x7fdcd8106720 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.66:42050/0 (socket says 172.21.15.66:42050) 2023-11-17T05:25:27.683 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:27.679+0000 7fdcde594700 1 -- 172.21.15.66:0/3394541902 learned_addr learned my addr 172.21.15.66:0/3394541902 (peer_addr_for_me v2:172.21.15.66:0/0) 2023-11-17T05:25:27.683 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:27.680+0000 7fdcde594700 1 -- 172.21.15.66:0/3394541902 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fdcc8005040 con 0x7fdcd80f5410 2023-11-17T05:25:27.683 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:27.680+0000 7fdcde594700 1 --2- 172.21.15.66:0/3394541902 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fdcd80f5410 0x7fdcd8106720 secure :-1 s=READY pgs=196 cs=0 l=1 rev1=1 rx=0x7fdcc8013510 tx=0x7fdcc800b040).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:25:27.683 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:27.680+0000 7fdcceffd700 1 -- 172.21.15.66:0/3394541902 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7fdcc8009320 con 0x7fdcd80f5410 2023-11-17T05:25:27.683 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:27.680+0000 7fdce07f8700 1 -- 172.21.15.66:0/3394541902 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fdcd8106e10 con 0x7fdcd80f5410 2023-11-17T05:25:27.683 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:27.680+0000 7fdcceffd700 1 -- 172.21.15.66:0/3394541902 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(25 keys) v1 ==== 1074+0+0 (secure 0 0 0) 0x7fdcc8013f00 con 0x7fdcd80f5410 2023-11-17T05:25:27.683 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:27.680+0000 7fdcceffd700 1 -- 172.21.15.66:0/3394541902 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7fdcc8014d90 con 0x7fdcd80f5410 2023-11-17T05:25:27.683 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:27.680+0000 7fdce07f8700 1 -- 172.21.15.66:0/3394541902 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fdcd81072a0 con 0x7fdcd80f5410 2023-11-17T05:25:27.683 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:27.681+0000 7fdcceffd700 1 -- 172.21.15.66:0/3394541902 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 17) v1 ==== 36269+0+0 (secure 0 0 0) 0x7fdcc80263a0 con 0x7fdcd80f5410 2023-11-17T05:25:27.683 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:27.681+0000 7fdcceffd700 1 --2- 172.21.15.66:0/3394541902 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fdcc402ff60 0x7fdcc4032420 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:25:27.684 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:27.681+0000 7fdcceffd700 1 -- 172.21.15.66:0/3394541902 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(5..5 src has 1..5) v4 ==== 1270+0+0 (secure 0 0 0) 0x7fdcc8041320 con 0x7fdcd80f5410 2023-11-17T05:25:27.684 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:27.681+0000 7fdce07f8700 1 -- 172.21.15.66:0/3394541902 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fdcd8059100 con 0x7fdcd80f5410 2023-11-17T05:25:27.684 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:27.681+0000 7fdcddd93700 1 --2- 172.21.15.66:0/3394541902 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fdcc402ff60 0x7fdcc4032420 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:25:27.684 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:27.681+0000 7fdcddd93700 1 --2- 172.21.15.66:0/3394541902 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fdcc402ff60 0x7fdcc4032420 secure :-1 s=READY pgs=63 cs=0 l=1 rev1=1 rx=0x7fdcd40096a0 tx=0x7fdcd4006b40).ready entity=mgr.14164 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:25:27.687 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:27.686+0000 7fdcceffd700 1 -- 172.21.15.66:0/3394541902 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+152212 (secure 0 0 0) 0x7fdcc8012920 con 0x7fdcd80f5410 2023-11-17T05:25:27.911 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:27.910+0000 7fdce07f8700 1 -- 172.21.15.66:0/3394541902 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "mon dump", "format": "json"} v 0) v1 -- 0x7fdcd80fae60 con 0x7fdcd80f5410 2023-11-17T05:25:27.912 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:27.910+0000 7fdcceffd700 1 -- 172.21.15.66:0/3394541902 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mon_command_ack([{"prefix": "mon dump", "format": "json"}]=0 dumped monmap epoch 1 v1) v1 ==== 95+0+690 (secure 0 0 0) 0x7fdcc802b030 con 0x7fdcd80f5410 2023-11-17T05:25:27.912 INFO:teuthology.orchestra.run.smithi066.stdout: 2023-11-17T05:25:27.912 INFO:teuthology.orchestra.run.smithi066.stdout:{"epoch":1,"fsid":"ed6470e2-8508-11ee-8dbe-212e2dc638e7","modified":"2023-11-17T05:20:13.934038Z","created":"2023-11-17T05:20:13.934038Z","min_mon_release":16,"min_mon_release_name":"pacific","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging"],"optional":[]},"mons":[{"rank":0,"name":"smithi040","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:3300","nonce":0},{"type":"v1","addr":"172.21.15.40:6789","nonce":0}]},"addr":"172.21.15.40:6789/0","public_addr":"172.21.15.40:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2023-11-17T05:25:27.913 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:27.912+0000 7fdce07f8700 1 -- 172.21.15.66:0/3394541902 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fdcc402ff60 msgr2=0x7fdcc4032420 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:25:27.913 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:27.912+0000 7fdce07f8700 1 --2- 172.21.15.66:0/3394541902 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fdcc402ff60 0x7fdcc4032420 secure :-1 s=READY pgs=63 cs=0 l=1 rev1=1 rx=0x7fdcd40096a0 tx=0x7fdcd4006b40).stop 2023-11-17T05:25:27.913 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:27.912+0000 7fdce07f8700 1 -- 172.21.15.66:0/3394541902 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fdcd80f5410 msgr2=0x7fdcd8106720 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:25:27.914 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:27.912+0000 7fdce07f8700 1 --2- 172.21.15.66:0/3394541902 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fdcd80f5410 0x7fdcd8106720 secure :-1 s=READY pgs=196 cs=0 l=1 rev1=1 rx=0x7fdcc8013510 tx=0x7fdcc800b040).stop 2023-11-17T05:25:27.914 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:27.912+0000 7fdce07f8700 1 -- 172.21.15.66:0/3394541902 shutdown_connections 2023-11-17T05:25:27.914 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:27.912+0000 7fdce07f8700 1 --2- 172.21.15.66:0/3394541902 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fdcc402ff60 0x7fdcc4032420 unknown :-1 s=CLOSED pgs=63 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:25:27.914 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:27.912+0000 7fdce07f8700 1 --2- 172.21.15.66:0/3394541902 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fdcd80f5410 0x7fdcd8106720 unknown :-1 s=CLOSED pgs=196 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:25:27.914 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:27.912+0000 7fdce07f8700 1 -- 172.21.15.66:0/3394541902 >> 172.21.15.66:0/3394541902 conn(0x7fdcd80f0b70 msgr2=0x7fdcd805fe60 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:25:27.914 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:27.912+0000 7fdce07f8700 1 -- 172.21.15.66:0/3394541902 shutdown_connections 2023-11-17T05:25:27.914 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:27.912+0000 7fdce07f8700 1 -- 172.21.15.66:0/3394541902 wait complete. 2023-11-17T05:25:27.914 INFO:teuthology.orchestra.run.smithi066.stderr:dumped monmap epoch 1 2023-11-17T05:25:28.739 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:25:28 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:25:27.196797+0000 mgr.smithi040.ogcprl (mgr.14164) 129 : cluster [DBG] pgmap v122: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:25:28.739 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:25:28 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:25:27.912178+0000 mon.smithi040 (mon.0) 179 : audit [DBG] from='client.? 172.21.15.66:0/3394541902' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-11-17T05:25:29.293 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2023-11-17T05:25:29.294 DEBUG:teuthology.orchestra.run.smithi066:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.4 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ed6470e2-8508-11ee-8dbe-212e2dc638e7 -- ceph mon dump -f json 2023-11-17T05:25:30.538 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:30.535+0000 7f97746e7700 1 -- 172.21.15.66:0/725737226 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f976c0f5410 msgr2=0x7f976c0f5830 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:25:30.538 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:30.535+0000 7f97746e7700 1 --2- 172.21.15.66:0/725737226 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f976c0f5410 0x7f976c0f5830 secure :-1 s=READY pgs=197 cs=0 l=1 rev1=1 rx=0x7f9768009fb0 tx=0x7f9768009b30).stop 2023-11-17T05:25:30.538 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:30.535+0000 7f97746e7700 1 -- 172.21.15.66:0/725737226 shutdown_connections 2023-11-17T05:25:30.538 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:30.535+0000 7f97746e7700 1 --2- 172.21.15.66:0/725737226 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f976c0f5410 0x7f976c0f5830 unknown :-1 s=CLOSED pgs=197 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:25:30.538 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:30.535+0000 7f97746e7700 1 -- 172.21.15.66:0/725737226 >> 172.21.15.66:0/725737226 conn(0x7f976c0f0b70 msgr2=0x7f976c0f2f90 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:25:30.538 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:30.535+0000 7f97746e7700 1 -- 172.21.15.66:0/725737226 shutdown_connections 2023-11-17T05:25:30.538 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:30.535+0000 7f97746e7700 1 -- 172.21.15.66:0/725737226 wait complete. 2023-11-17T05:25:30.538 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:30.536+0000 7f97746e7700 1 Processor -- start 2023-11-17T05:25:30.538 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:30.536+0000 7f97746e7700 1 -- start start 2023-11-17T05:25:30.538 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:30.536+0000 7f97746e7700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f976c0f5410 0x7f976c1093c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:25:30.538 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:30.536+0000 7f97746e7700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f976c060280 con 0x7f976c0f5410 2023-11-17T05:25:30.539 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:30.536+0000 7f9772483700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f976c0f5410 0x7f976c1093c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:25:30.539 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:30.536+0000 7f9772483700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f976c0f5410 0x7f976c1093c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.66:42066/0 (socket says 172.21.15.66:42066) 2023-11-17T05:25:30.539 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:30.536+0000 7f9772483700 1 -- 172.21.15.66:0/3560862415 learned_addr learned my addr 172.21.15.66:0/3560862415 (peer_addr_for_me v2:172.21.15.66:0/0) 2023-11-17T05:25:30.539 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:30.537+0000 7f9772483700 1 -- 172.21.15.66:0/3560862415 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f9768005040 con 0x7f976c0f5410 2023-11-17T05:25:30.540 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:30.537+0000 7f9772483700 1 --2- 172.21.15.66:0/3560862415 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f976c0f5410 0x7f976c1093c0 secure :-1 s=READY pgs=198 cs=0 l=1 rev1=1 rx=0x7f9768013510 tx=0x7f976800b040).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:25:30.540 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:30.537+0000 7f975effd700 1 -- 172.21.15.66:0/3560862415 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f9768009320 con 0x7f976c0f5410 2023-11-17T05:25:30.540 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:30.537+0000 7f97746e7700 1 -- 172.21.15.66:0/3560862415 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f976c060480 con 0x7f976c0f5410 2023-11-17T05:25:30.540 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:30.537+0000 7f975effd700 1 -- 172.21.15.66:0/3560862415 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(25 keys) v1 ==== 1074+0+0 (secure 0 0 0) 0x7f9768013f00 con 0x7f976c0f5410 2023-11-17T05:25:30.540 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:30.537+0000 7f975effd700 1 -- 172.21.15.66:0/3560862415 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f9768014d90 con 0x7f976c0f5410 2023-11-17T05:25:30.540 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:30.537+0000 7f97746e7700 1 -- 172.21.15.66:0/3560862415 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f976c109af0 con 0x7f976c0f5410 2023-11-17T05:25:30.540 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:30.538+0000 7f975effd700 1 -- 172.21.15.66:0/3560862415 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 17) v1 ==== 36269+0+0 (secure 0 0 0) 0x7f97680095f0 con 0x7f976c0f5410 2023-11-17T05:25:30.541 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:30.538+0000 7f97746e7700 1 -- 172.21.15.66:0/3560862415 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f976c045ad0 con 0x7f976c0f5410 2023-11-17T05:25:30.541 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:30.538+0000 7f975effd700 1 --2- 172.21.15.66:0/3560862415 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f975802ff60 0x7f9758032420 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:25:30.541 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:30.538+0000 7f975effd700 1 -- 172.21.15.66:0/3560862415 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(5..5 src has 1..5) v4 ==== 1270+0+0 (secure 0 0 0) 0x7f9768041000 con 0x7f976c0f5410 2023-11-17T05:25:30.541 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:30.538+0000 7f9771c82700 1 --2- 172.21.15.66:0/3560862415 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f975802ff60 0x7f9758032420 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:25:30.541 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:30.539+0000 7f9771c82700 1 --2- 172.21.15.66:0/3560862415 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f975802ff60 0x7f9758032420 secure :-1 s=READY pgs=64 cs=0 l=1 rev1=1 rx=0x7f97600096a0 tx=0x7f9760006b40).ready entity=mgr.14164 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:25:30.544 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:30.543+0000 7f975effd700 1 -- 172.21.15.66:0/3560862415 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+152212 (secure 0 0 0) 0x7f9768014420 con 0x7f976c0f5410 2023-11-17T05:25:30.744 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:25:30 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:25:29.197024+0000 mgr.smithi040.ogcprl (mgr.14164) 130 : cluster [DBG] pgmap v123: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:25:30.761 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:30.759+0000 7f97746e7700 1 -- 172.21.15.66:0/3560862415 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "mon dump", "format": "json"} v 0) v1 -- 0x7f976c10a310 con 0x7f976c0f5410 2023-11-17T05:25:30.761 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:30.760+0000 7f975effd700 1 -- 172.21.15.66:0/3560862415 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mon_command_ack([{"prefix": "mon dump", "format": "json"}]=0 dumped monmap epoch 1 v1) v1 ==== 95+0+690 (secure 0 0 0) 0x7f9768010230 con 0x7f976c0f5410 2023-11-17T05:25:30.761 INFO:teuthology.orchestra.run.smithi066.stdout: 2023-11-17T05:25:30.761 INFO:teuthology.orchestra.run.smithi066.stdout:{"epoch":1,"fsid":"ed6470e2-8508-11ee-8dbe-212e2dc638e7","modified":"2023-11-17T05:20:13.934038Z","created":"2023-11-17T05:20:13.934038Z","min_mon_release":16,"min_mon_release_name":"pacific","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging"],"optional":[]},"mons":[{"rank":0,"name":"smithi040","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:3300","nonce":0},{"type":"v1","addr":"172.21.15.40:6789","nonce":0}]},"addr":"172.21.15.40:6789/0","public_addr":"172.21.15.40:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2023-11-17T05:25:30.763 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:30.761+0000 7f97746e7700 1 -- 172.21.15.66:0/3560862415 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f975802ff60 msgr2=0x7f9758032420 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:25:30.763 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:30.761+0000 7f97746e7700 1 --2- 172.21.15.66:0/3560862415 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f975802ff60 0x7f9758032420 secure :-1 s=READY pgs=64 cs=0 l=1 rev1=1 rx=0x7f97600096a0 tx=0x7f9760006b40).stop 2023-11-17T05:25:30.763 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:30.761+0000 7f97746e7700 1 -- 172.21.15.66:0/3560862415 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f976c0f5410 msgr2=0x7f976c1093c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:25:30.763 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:30.761+0000 7f97746e7700 1 --2- 172.21.15.66:0/3560862415 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f976c0f5410 0x7f976c1093c0 secure :-1 s=READY pgs=198 cs=0 l=1 rev1=1 rx=0x7f9768013510 tx=0x7f976800b040).stop 2023-11-17T05:25:30.763 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:30.761+0000 7f97746e7700 1 -- 172.21.15.66:0/3560862415 shutdown_connections 2023-11-17T05:25:30.763 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:30.761+0000 7f97746e7700 1 --2- 172.21.15.66:0/3560862415 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f975802ff60 0x7f9758032420 unknown :-1 s=CLOSED pgs=64 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:25:30.763 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:30.761+0000 7f97746e7700 1 --2- 172.21.15.66:0/3560862415 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f976c0f5410 0x7f976c1093c0 unknown :-1 s=CLOSED pgs=198 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:25:30.763 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:30.761+0000 7f97746e7700 1 -- 172.21.15.66:0/3560862415 >> 172.21.15.66:0/3560862415 conn(0x7f976c0f0b70 msgr2=0x7f976c0f1630 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:25:30.763 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:30.762+0000 7f97746e7700 1 -- 172.21.15.66:0/3560862415 shutdown_connections 2023-11-17T05:25:30.763 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:30.762+0000 7f97746e7700 1 -- 172.21.15.66:0/3560862415 wait complete. 2023-11-17T05:25:30.764 INFO:teuthology.orchestra.run.smithi066.stderr:dumped monmap epoch 1 2023-11-17T05:25:31.739 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:25:31 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:25:30.761614+0000 mon.smithi040 (mon.0) 180 : audit [DBG] from='client.? 172.21.15.66:0/3560862415' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-11-17T05:25:32.168 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2023-11-17T05:25:32.168 DEBUG:teuthology.orchestra.run.smithi066:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.4 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ed6470e2-8508-11ee-8dbe-212e2dc638e7 -- ceph mon dump -f json 2023-11-17T05:25:32.741 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:25:32 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:25:31.197291+0000 mgr.smithi040.ogcprl (mgr.14164) 131 : cluster [DBG] pgmap v124: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:25:33.408 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:33.406+0000 7ff65aa2f700 1 -- 172.21.15.66:0/488306627 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7ff6540f62a0 msgr2=0x7ff6540f66c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:25:33.408 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:33.406+0000 7ff65aa2f700 1 --2- 172.21.15.66:0/488306627 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7ff6540f62a0 0x7ff6540f66c0 secure :-1 s=READY pgs=199 cs=0 l=1 rev1=1 rx=0x7ff650009960 tx=0x7ff650004d10).stop 2023-11-17T05:25:33.408 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:33.406+0000 7ff65aa2f700 1 -- 172.21.15.66:0/488306627 shutdown_connections 2023-11-17T05:25:33.408 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:33.406+0000 7ff65aa2f700 1 --2- 172.21.15.66:0/488306627 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7ff6540f62a0 0x7ff6540f66c0 unknown :-1 s=CLOSED pgs=199 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:25:33.409 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:33.406+0000 7ff65aa2f700 1 -- 172.21.15.66:0/488306627 >> 172.21.15.66:0/488306627 conn(0x7ff6540f1ac0 msgr2=0x7ff6540f3ee0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:25:33.409 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:33.407+0000 7ff65aa2f700 1 -- 172.21.15.66:0/488306627 shutdown_connections 2023-11-17T05:25:33.409 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:33.407+0000 7ff65aa2f700 1 -- 172.21.15.66:0/488306627 wait complete. 2023-11-17T05:25:33.411 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:33.407+0000 7ff65aa2f700 1 Processor -- start 2023-11-17T05:25:33.411 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:33.407+0000 7ff65aa2f700 1 -- start start 2023-11-17T05:25:33.411 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:33.407+0000 7ff65aa2f700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7ff6540f62a0 0x7ff654106500 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:25:33.411 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:33.407+0000 7ff65aa2f700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7ff6541069f0 con 0x7ff6540f62a0 2023-11-17T05:25:33.411 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:33.408+0000 7ff659a2d700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7ff6540f62a0 0x7ff654106500 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:25:33.411 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:33.408+0000 7ff659a2d700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7ff6540f62a0 0x7ff654106500 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.66:36752/0 (socket says 172.21.15.66:36752) 2023-11-17T05:25:33.411 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:33.408+0000 7ff659a2d700 1 -- 172.21.15.66:0/3057709659 learned_addr learned my addr 172.21.15.66:0/3057709659 (peer_addr_for_me v2:172.21.15.66:0/0) 2023-11-17T05:25:33.411 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:33.408+0000 7ff659a2d700 1 -- 172.21.15.66:0/3057709659 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7ff650009640 con 0x7ff6540f62a0 2023-11-17T05:25:33.411 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:33.408+0000 7ff659a2d700 1 --2- 172.21.15.66:0/3057709659 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7ff6540f62a0 0x7ff654106500 secure :-1 s=READY pgs=200 cs=0 l=1 rev1=1 rx=0x7ff650007bd0 tx=0x7ff650004d10).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:25:33.411 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:33.408+0000 7ff64a7fc700 1 -- 172.21.15.66:0/3057709659 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7ff650018070 con 0x7ff6540f62a0 2023-11-17T05:25:33.411 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:33.408+0000 7ff65aa2f700 1 -- 172.21.15.66:0/3057709659 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7ff654106bf0 con 0x7ff6540f62a0 2023-11-17T05:25:33.411 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:33.408+0000 7ff64a7fc700 1 -- 172.21.15.66:0/3057709659 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(25 keys) v1 ==== 1074+0+0 (secure 0 0 0) 0x7ff65000b870 con 0x7ff6540f62a0 2023-11-17T05:25:33.411 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:33.408+0000 7ff64a7fc700 1 -- 172.21.15.66:0/3057709659 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7ff65001d420 con 0x7ff6540f62a0 2023-11-17T05:25:33.412 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:33.408+0000 7ff65aa2f700 1 -- 172.21.15.66:0/3057709659 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7ff654107080 con 0x7ff6540f62a0 2023-11-17T05:25:33.412 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:33.409+0000 7ff64a7fc700 1 -- 172.21.15.66:0/3057709659 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 17) v1 ==== 36269+0+0 (secure 0 0 0) 0x7ff650024060 con 0x7ff6540f62a0 2023-11-17T05:25:33.412 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:33.409+0000 7ff64a7fc700 1 --2- 172.21.15.66:0/3057709659 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7ff64002ff60 0x7ff640032420 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:25:33.412 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:33.409+0000 7ff64a7fc700 1 -- 172.21.15.66:0/3057709659 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(5..5 src has 1..5) v4 ==== 1270+0+0 (secure 0 0 0) 0x7ff6500411e0 con 0x7ff6540f62a0 2023-11-17T05:25:33.412 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:33.409+0000 7ff65aa2f700 1 -- 172.21.15.66:0/3057709659 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7ff654045ad0 con 0x7ff6540f62a0 2023-11-17T05:25:33.412 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:33.409+0000 7ff65922c700 1 --2- 172.21.15.66:0/3057709659 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7ff64002ff60 0x7ff640032420 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:25:33.412 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:33.410+0000 7ff65922c700 1 --2- 172.21.15.66:0/3057709659 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7ff64002ff60 0x7ff640032420 secure :-1 s=READY pgs=65 cs=0 l=1 rev1=1 rx=0x7ff6440096a0 tx=0x7ff644006b40).ready entity=mgr.14164 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:25:33.415 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:33.414+0000 7ff64a7fc700 1 -- 172.21.15.66:0/3057709659 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+152212 (secure 0 0 0) 0x7ff650025340 con 0x7ff6540f62a0 2023-11-17T05:25:33.634 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:33.632+0000 7ff65aa2f700 1 -- 172.21.15.66:0/3057709659 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "mon dump", "format": "json"} v 0) v1 -- 0x7ff654107d30 con 0x7ff6540f62a0 2023-11-17T05:25:33.634 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:33.634+0000 7ff64a7fc700 1 -- 172.21.15.66:0/3057709659 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mon_command_ack([{"prefix": "mon dump", "format": "json"}]=0 dumped monmap epoch 1 v1) v1 ==== 95+0+690 (secure 0 0 0) 0x7ff6500048f0 con 0x7ff6540f62a0 2023-11-17T05:25:33.635 INFO:teuthology.orchestra.run.smithi066.stdout: 2023-11-17T05:25:33.635 INFO:teuthology.orchestra.run.smithi066.stdout:{"epoch":1,"fsid":"ed6470e2-8508-11ee-8dbe-212e2dc638e7","modified":"2023-11-17T05:20:13.934038Z","created":"2023-11-17T05:20:13.934038Z","min_mon_release":16,"min_mon_release_name":"pacific","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging"],"optional":[]},"mons":[{"rank":0,"name":"smithi040","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:3300","nonce":0},{"type":"v1","addr":"172.21.15.40:6789","nonce":0}]},"addr":"172.21.15.40:6789/0","public_addr":"172.21.15.40:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2023-11-17T05:25:33.636 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:33.635+0000 7ff65aa2f700 1 -- 172.21.15.66:0/3057709659 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7ff64002ff60 msgr2=0x7ff640032420 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:25:33.636 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:33.635+0000 7ff65aa2f700 1 --2- 172.21.15.66:0/3057709659 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7ff64002ff60 0x7ff640032420 secure :-1 s=READY pgs=65 cs=0 l=1 rev1=1 rx=0x7ff6440096a0 tx=0x7ff644006b40).stop 2023-11-17T05:25:33.637 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:33.635+0000 7ff65aa2f700 1 -- 172.21.15.66:0/3057709659 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7ff6540f62a0 msgr2=0x7ff654106500 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:25:33.637 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:33.635+0000 7ff65aa2f700 1 --2- 172.21.15.66:0/3057709659 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7ff6540f62a0 0x7ff654106500 secure :-1 s=READY pgs=200 cs=0 l=1 rev1=1 rx=0x7ff650007bd0 tx=0x7ff650004d10).stop 2023-11-17T05:25:33.637 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:33.635+0000 7ff65aa2f700 1 -- 172.21.15.66:0/3057709659 shutdown_connections 2023-11-17T05:25:33.637 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:33.635+0000 7ff65aa2f700 1 --2- 172.21.15.66:0/3057709659 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7ff64002ff60 0x7ff640032420 unknown :-1 s=CLOSED pgs=65 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:25:33.637 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:33.635+0000 7ff65aa2f700 1 --2- 172.21.15.66:0/3057709659 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7ff6540f62a0 0x7ff654106500 unknown :-1 s=CLOSED pgs=200 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:25:33.637 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:33.635+0000 7ff65aa2f700 1 -- 172.21.15.66:0/3057709659 >> 172.21.15.66:0/3057709659 conn(0x7ff6540f1ac0 msgr2=0x7ff6540f2530 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:25:33.638 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:33.635+0000 7ff65aa2f700 1 -- 172.21.15.66:0/3057709659 shutdown_connections 2023-11-17T05:25:33.638 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:33.635+0000 7ff65aa2f700 1 -- 172.21.15.66:0/3057709659 wait complete. 2023-11-17T05:25:33.638 INFO:teuthology.orchestra.run.smithi066.stderr:dumped monmap epoch 1 2023-11-17T05:25:34.739 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:25:34 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:25:33.197607+0000 mgr.smithi040.ogcprl (mgr.14164) 132 : cluster [DBG] pgmap v125: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:25:34.739 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:25:34 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:25:33.634689+0000 mon.smithi040 (mon.0) 181 : audit [DBG] from='client.? 172.21.15.66:0/3057709659' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-11-17T05:25:35.061 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2023-11-17T05:25:35.062 DEBUG:teuthology.orchestra.run.smithi066:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.4 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ed6470e2-8508-11ee-8dbe-212e2dc638e7 -- ceph mon dump -f json 2023-11-17T05:25:36.283 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:36.280+0000 7fa7036ec700 1 -- 172.21.15.66:0/1797016040 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fa6fc0f5410 msgr2=0x7fa6fc0f5830 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:25:36.284 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:36.280+0000 7fa7036ec700 1 --2- 172.21.15.66:0/1797016040 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fa6fc0f5410 0x7fa6fc0f5830 secure :-1 s=READY pgs=201 cs=0 l=1 rev1=1 rx=0x7fa6f8009fb0 tx=0x7fa6f8009b30).stop 2023-11-17T05:25:36.284 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:36.280+0000 7fa7036ec700 1 -- 172.21.15.66:0/1797016040 shutdown_connections 2023-11-17T05:25:36.284 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:36.280+0000 7fa7036ec700 1 --2- 172.21.15.66:0/1797016040 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fa6fc0f5410 0x7fa6fc0f5830 unknown :-1 s=CLOSED pgs=201 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:25:36.284 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:36.280+0000 7fa7036ec700 1 -- 172.21.15.66:0/1797016040 >> 172.21.15.66:0/1797016040 conn(0x7fa6fc0f0b70 msgr2=0x7fa6fc0f2f90 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:25:36.284 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:36.280+0000 7fa7036ec700 1 -- 172.21.15.66:0/1797016040 shutdown_connections 2023-11-17T05:25:36.284 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:36.280+0000 7fa7036ec700 1 -- 172.21.15.66:0/1797016040 wait complete. 2023-11-17T05:25:36.284 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:36.281+0000 7fa7036ec700 1 Processor -- start 2023-11-17T05:25:36.284 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:36.281+0000 7fa7036ec700 1 -- start start 2023-11-17T05:25:36.284 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:36.281+0000 7fa7036ec700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fa6fc0f5410 0x7fa6fc109450 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:25:36.285 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:36.281+0000 7fa7036ec700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fa6fc060220 con 0x7fa6fc0f5410 2023-11-17T05:25:36.285 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:36.281+0000 7fa701488700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fa6fc0f5410 0x7fa6fc109450 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:25:36.285 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:36.281+0000 7fa701488700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fa6fc0f5410 0x7fa6fc109450 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.66:36770/0 (socket says 172.21.15.66:36770) 2023-11-17T05:25:36.285 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:36.281+0000 7fa701488700 1 -- 172.21.15.66:0/773546775 learned_addr learned my addr 172.21.15.66:0/773546775 (peer_addr_for_me v2:172.21.15.66:0/0) 2023-11-17T05:25:36.285 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:36.281+0000 7fa701488700 1 -- 172.21.15.66:0/773546775 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fa6f8005040 con 0x7fa6fc0f5410 2023-11-17T05:25:36.285 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:36.282+0000 7fa701488700 1 --2- 172.21.15.66:0/773546775 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fa6fc0f5410 0x7fa6fc109450 secure :-1 s=READY pgs=202 cs=0 l=1 rev1=1 rx=0x7fa6f8014000 tx=0x7fa6f800b040).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:25:36.285 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:36.282+0000 7fa6edffb700 1 -- 172.21.15.66:0/773546775 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7fa6f80095f0 con 0x7fa6fc0f5410 2023-11-17T05:25:36.285 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:36.282+0000 7fa7036ec700 1 -- 172.21.15.66:0/773546775 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fa6fc060420 con 0x7fa6fc0f5410 2023-11-17T05:25:36.285 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:36.282+0000 7fa6edffb700 1 -- 172.21.15.66:0/773546775 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(25 keys) v1 ==== 1074+0+0 (secure 0 0 0) 0x7fa6f8007e70 con 0x7fa6fc0f5410 2023-11-17T05:25:36.285 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:36.282+0000 7fa6edffb700 1 -- 172.21.15.66:0/773546775 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7fa6f8014ea0 con 0x7fa6fc0f5410 2023-11-17T05:25:36.285 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:36.282+0000 7fa7036ec700 1 -- 172.21.15.66:0/773546775 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fa6fc109b00 con 0x7fa6fc0f5410 2023-11-17T05:25:36.286 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:36.283+0000 7fa6edffb700 1 -- 172.21.15.66:0/773546775 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 17) v1 ==== 36269+0+0 (secure 0 0 0) 0x7fa6f8025070 con 0x7fa6fc0f5410 2023-11-17T05:25:36.286 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:36.283+0000 7fa6edffb700 1 --2- 172.21.15.66:0/773546775 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fa6e802ff60 0x7fa6e8032420 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:25:36.286 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:36.283+0000 7fa6edffb700 1 -- 172.21.15.66:0/773546775 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(5..5 src has 1..5) v4 ==== 1270+0+0 (secure 0 0 0) 0x7fa6f8040a60 con 0x7fa6fc0f5410 2023-11-17T05:25:36.286 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:36.283+0000 7fa7036ec700 1 -- 172.21.15.66:0/773546775 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fa6fc059100 con 0x7fa6fc0f5410 2023-11-17T05:25:36.286 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:36.283+0000 7fa700c87700 1 --2- 172.21.15.66:0/773546775 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fa6e802ff60 0x7fa6e8032420 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:25:36.287 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:36.284+0000 7fa700c87700 1 --2- 172.21.15.66:0/773546775 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fa6e802ff60 0x7fa6e8032420 secure :-1 s=READY pgs=66 cs=0 l=1 rev1=1 rx=0x7fa6f00096a0 tx=0x7fa6f0006b40).ready entity=mgr.14164 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:25:36.289 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:36.288+0000 7fa6edffb700 1 -- 172.21.15.66:0/773546775 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+152212 (secure 0 0 0) 0x7fa6f8026350 con 0x7fa6fc0f5410 2023-11-17T05:25:36.510 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:36.508+0000 7fa7036ec700 1 -- 172.21.15.66:0/773546775 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "mon dump", "format": "json"} v 0) v1 -- 0x7fa6fc10a250 con 0x7fa6fc0f5410 2023-11-17T05:25:36.510 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:36.509+0000 7fa6edffb700 1 -- 172.21.15.66:0/773546775 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mon_command_ack([{"prefix": "mon dump", "format": "json"}]=0 dumped monmap epoch 1 v1) v1 ==== 95+0+690 (secure 0 0 0) 0x7fa6f8012330 con 0x7fa6fc0f5410 2023-11-17T05:25:36.511 INFO:teuthology.orchestra.run.smithi066.stdout: 2023-11-17T05:25:36.511 INFO:teuthology.orchestra.run.smithi066.stdout:{"epoch":1,"fsid":"ed6470e2-8508-11ee-8dbe-212e2dc638e7","modified":"2023-11-17T05:20:13.934038Z","created":"2023-11-17T05:20:13.934038Z","min_mon_release":16,"min_mon_release_name":"pacific","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging"],"optional":[]},"mons":[{"rank":0,"name":"smithi040","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:3300","nonce":0},{"type":"v1","addr":"172.21.15.40:6789","nonce":0}]},"addr":"172.21.15.40:6789/0","public_addr":"172.21.15.40:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2023-11-17T05:25:36.512 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:36.511+0000 7fa7036ec700 1 -- 172.21.15.66:0/773546775 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fa6e802ff60 msgr2=0x7fa6e8032420 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:25:36.513 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:36.511+0000 7fa7036ec700 1 --2- 172.21.15.66:0/773546775 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fa6e802ff60 0x7fa6e8032420 secure :-1 s=READY pgs=66 cs=0 l=1 rev1=1 rx=0x7fa6f00096a0 tx=0x7fa6f0006b40).stop 2023-11-17T05:25:36.513 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:36.511+0000 7fa7036ec700 1 -- 172.21.15.66:0/773546775 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fa6fc0f5410 msgr2=0x7fa6fc109450 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:25:36.513 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:36.511+0000 7fa7036ec700 1 --2- 172.21.15.66:0/773546775 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fa6fc0f5410 0x7fa6fc109450 secure :-1 s=READY pgs=202 cs=0 l=1 rev1=1 rx=0x7fa6f8014000 tx=0x7fa6f800b040).stop 2023-11-17T05:25:36.513 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:36.511+0000 7fa7036ec700 1 -- 172.21.15.66:0/773546775 shutdown_connections 2023-11-17T05:25:36.513 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:36.511+0000 7fa7036ec700 1 --2- 172.21.15.66:0/773546775 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fa6e802ff60 0x7fa6e8032420 unknown :-1 s=CLOSED pgs=66 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:25:36.513 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:36.511+0000 7fa7036ec700 1 --2- 172.21.15.66:0/773546775 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fa6fc0f5410 0x7fa6fc109450 unknown :-1 s=CLOSED pgs=202 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:25:36.513 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:36.511+0000 7fa7036ec700 1 -- 172.21.15.66:0/773546775 >> 172.21.15.66:0/773546775 conn(0x7fa6fc0f0b70 msgr2=0x7fa6fc0f1660 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:25:36.513 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:36.511+0000 7fa7036ec700 1 -- 172.21.15.66:0/773546775 shutdown_connections 2023-11-17T05:25:36.513 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:36.511+0000 7fa7036ec700 1 -- 172.21.15.66:0/773546775 wait complete. 2023-11-17T05:25:36.513 INFO:teuthology.orchestra.run.smithi066.stderr:dumped monmap epoch 1 2023-11-17T05:25:36.739 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:25:36 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:25:35.198004+0000 mgr.smithi040.ogcprl (mgr.14164) 133 : cluster [DBG] pgmap v126: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:25:37.739 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:25:37 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:25:36.510791+0000 mon.smithi040 (mon.0) 182 : audit [DBG] from='client.? 172.21.15.66:0/773546775' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-11-17T05:25:37.902 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2023-11-17T05:25:37.902 DEBUG:teuthology.orchestra.run.smithi066:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.4 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ed6470e2-8508-11ee-8dbe-212e2dc638e7 -- ceph mon dump -f json 2023-11-17T05:25:38.739 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:25:38 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:25:37.198316+0000 mgr.smithi040.ogcprl (mgr.14164) 134 : cluster [DBG] pgmap v127: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:25:39.054 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:39.052+0000 7f74c49bb700 1 -- 172.21.15.66:0/1948431754 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f74c00f4740 msgr2=0x7f74c00f4b60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:25:39.054 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:39.052+0000 7f74c49bb700 1 --2- 172.21.15.66:0/1948431754 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f74c00f4740 0x7f74c00f4b60 secure :-1 s=READY pgs=203 cs=0 l=1 rev1=1 rx=0x7f74b0009fb0 tx=0x7f74b0009b30).stop 2023-11-17T05:25:39.054 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:39.053+0000 7f74c49bb700 1 -- 172.21.15.66:0/1948431754 shutdown_connections 2023-11-17T05:25:39.054 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:39.053+0000 7f74c49bb700 1 --2- 172.21.15.66:0/1948431754 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f74c00f4740 0x7f74c00f4b60 unknown :-1 s=CLOSED pgs=203 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:25:39.054 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:39.053+0000 7f74c49bb700 1 -- 172.21.15.66:0/1948431754 >> 172.21.15.66:0/1948431754 conn(0x7f74c00f0260 msgr2=0x7f74c00f2680 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:25:39.054 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:39.053+0000 7f74c49bb700 1 -- 172.21.15.66:0/1948431754 shutdown_connections 2023-11-17T05:25:39.055 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:39.053+0000 7f74c49bb700 1 -- 172.21.15.66:0/1948431754 wait complete. 2023-11-17T05:25:39.056 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:39.053+0000 7f74c49bb700 1 Processor -- start 2023-11-17T05:25:39.056 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:39.053+0000 7f74c49bb700 1 -- start start 2023-11-17T05:25:39.056 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:39.054+0000 7f74c49bb700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f74c00f4740 0x7f74c01066d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:25:39.056 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:39.054+0000 7f74c49bb700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f74c0106bc0 con 0x7f74c00f4740 2023-11-17T05:25:39.056 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:39.054+0000 7f74be59c700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f74c00f4740 0x7f74c01066d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:25:39.056 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:39.054+0000 7f74be59c700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f74c00f4740 0x7f74c01066d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.66:36796/0 (socket says 172.21.15.66:36796) 2023-11-17T05:25:39.056 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:39.054+0000 7f74be59c700 1 -- 172.21.15.66:0/3602896046 learned_addr learned my addr 172.21.15.66:0/3602896046 (peer_addr_for_me v2:172.21.15.66:0/0) 2023-11-17T05:25:39.056 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:39.054+0000 7f74be59c700 1 -- 172.21.15.66:0/3602896046 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f74b0005040 con 0x7f74c00f4740 2023-11-17T05:25:39.056 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:39.054+0000 7f74be59c700 1 --2- 172.21.15.66:0/3602896046 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f74c00f4740 0x7f74c01066d0 secure :-1 s=READY pgs=204 cs=0 l=1 rev1=1 rx=0x7f74b0013510 tx=0x7f74b000b040).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:25:39.057 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:39.054+0000 7f74b6ffd700 1 -- 172.21.15.66:0/3602896046 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f74b0009320 con 0x7f74c00f4740 2023-11-17T05:25:39.057 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:39.054+0000 7f74c49bb700 1 -- 172.21.15.66:0/3602896046 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f74c0106dc0 con 0x7f74c00f4740 2023-11-17T05:25:39.057 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:39.054+0000 7f74b6ffd700 1 -- 172.21.15.66:0/3602896046 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(25 keys) v1 ==== 1074+0+0 (secure 0 0 0) 0x7f74b0013f00 con 0x7f74c00f4740 2023-11-17T05:25:39.057 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:39.054+0000 7f74b6ffd700 1 -- 172.21.15.66:0/3602896046 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f74b0014d90 con 0x7f74c00f4740 2023-11-17T05:25:39.057 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:39.054+0000 7f74c49bb700 1 -- 172.21.15.66:0/3602896046 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f74c0107250 con 0x7f74c00f4740 2023-11-17T05:25:39.058 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:39.055+0000 7f74b6ffd700 1 -- 172.21.15.66:0/3602896046 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 17) v1 ==== 36269+0+0 (secure 0 0 0) 0x7f74b0014420 con 0x7f74c00f4740 2023-11-17T05:25:39.058 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:39.055+0000 7f74c49bb700 1 -- 172.21.15.66:0/3602896046 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f74c0045ad0 con 0x7f74c00f4740 2023-11-17T05:25:39.058 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:39.055+0000 7f74b6ffd700 1 --2- 172.21.15.66:0/3602896046 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f74ac02ffb0 0x7f74ac032470 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:25:39.058 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:39.055+0000 7f74b6ffd700 1 -- 172.21.15.66:0/3602896046 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(5..5 src has 1..5) v4 ==== 1270+0+0 (secure 0 0 0) 0x7f74b00410b0 con 0x7f74c00f4740 2023-11-17T05:25:39.058 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:39.056+0000 7f74bdd9b700 1 --2- 172.21.15.66:0/3602896046 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f74ac02ffb0 0x7f74ac032470 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:25:39.058 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:39.056+0000 7f74bdd9b700 1 --2- 172.21.15.66:0/3602896046 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f74ac02ffb0 0x7f74ac032470 secure :-1 s=READY pgs=67 cs=0 l=1 rev1=1 rx=0x7f74a80096a0 tx=0x7f74a8006b40).ready entity=mgr.14164 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:25:39.067 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:39.061+0000 7f74b6ffd700 1 -- 172.21.15.66:0/3602896046 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+152212 (secure 0 0 0) 0x7f74b00095f0 con 0x7f74c00f4740 2023-11-17T05:25:39.278 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:39.277+0000 7f74c49bb700 1 -- 172.21.15.66:0/3602896046 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "mon dump", "format": "json"} v 0) v1 -- 0x7f74c0108100 con 0x7f74c00f4740 2023-11-17T05:25:39.279 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:39.278+0000 7f74b6ffd700 1 -- 172.21.15.66:0/3602896046 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mon_command_ack([{"prefix": "mon dump", "format": "json"}]=0 dumped monmap epoch 1 v1) v1 ==== 95+0+690 (secure 0 0 0) 0x7f74b001db70 con 0x7f74c00f4740 2023-11-17T05:25:39.279 INFO:teuthology.orchestra.run.smithi066.stdout: 2023-11-17T05:25:39.279 INFO:teuthology.orchestra.run.smithi066.stdout:{"epoch":1,"fsid":"ed6470e2-8508-11ee-8dbe-212e2dc638e7","modified":"2023-11-17T05:20:13.934038Z","created":"2023-11-17T05:20:13.934038Z","min_mon_release":16,"min_mon_release_name":"pacific","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging"],"optional":[]},"mons":[{"rank":0,"name":"smithi040","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:3300","nonce":0},{"type":"v1","addr":"172.21.15.40:6789","nonce":0}]},"addr":"172.21.15.40:6789/0","public_addr":"172.21.15.40:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2023-11-17T05:25:39.281 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:39.279+0000 7f74c49bb700 1 -- 172.21.15.66:0/3602896046 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f74ac02ffb0 msgr2=0x7f74ac032470 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:25:39.281 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:39.279+0000 7f74c49bb700 1 --2- 172.21.15.66:0/3602896046 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f74ac02ffb0 0x7f74ac032470 secure :-1 s=READY pgs=67 cs=0 l=1 rev1=1 rx=0x7f74a80096a0 tx=0x7f74a8006b40).stop 2023-11-17T05:25:39.281 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:39.279+0000 7f74c49bb700 1 -- 172.21.15.66:0/3602896046 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f74c00f4740 msgr2=0x7f74c01066d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:25:39.281 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:39.279+0000 7f74c49bb700 1 --2- 172.21.15.66:0/3602896046 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f74c00f4740 0x7f74c01066d0 secure :-1 s=READY pgs=204 cs=0 l=1 rev1=1 rx=0x7f74b0013510 tx=0x7f74b000b040).stop 2023-11-17T05:25:39.281 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:39.279+0000 7f74c49bb700 1 -- 172.21.15.66:0/3602896046 shutdown_connections 2023-11-17T05:25:39.281 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:39.279+0000 7f74c49bb700 1 --2- 172.21.15.66:0/3602896046 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f74ac02ffb0 0x7f74ac032470 unknown :-1 s=CLOSED pgs=67 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:25:39.281 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:39.279+0000 7f74c49bb700 1 --2- 172.21.15.66:0/3602896046 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f74c00f4740 0x7f74c01066d0 unknown :-1 s=CLOSED pgs=204 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:25:39.282 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:39.279+0000 7f74c49bb700 1 -- 172.21.15.66:0/3602896046 >> 172.21.15.66:0/3602896046 conn(0x7f74c00f0260 msgr2=0x7f74c00f0d80 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:25:39.282 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:39.280+0000 7f74c49bb700 1 -- 172.21.15.66:0/3602896046 shutdown_connections 2023-11-17T05:25:39.282 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:39.280+0000 7f74c49bb700 1 -- 172.21.15.66:0/3602896046 wait complete. 2023-11-17T05:25:39.282 INFO:teuthology.orchestra.run.smithi066.stderr:dumped monmap epoch 1 2023-11-17T05:25:39.739 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:25:39 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:25:39.279465+0000 mon.smithi040 (mon.0) 183 : audit [DBG] from='client.? 172.21.15.66:0/3602896046' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-11-17T05:25:40.658 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2023-11-17T05:25:47.816 DEBUG:teuthology.orchestra.run.smithi066:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.4 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ed6470e2-8508-11ee-8dbe-212e2dc638e7 -- ceph mon dump -f json 2023-11-17T05:25:47.818 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:25:40 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:25:39.198646+0000 mgr.smithi040.ogcprl (mgr.14164) 135 : cluster [DBG] pgmap v128: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:25:47.818 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:25:42 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:25:41.198980+0000 mgr.smithi040.ogcprl (mgr.14164) 136 : cluster [DBG] pgmap v129: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:25:47.818 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:25:44 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:25:43.199406+0000 mgr.smithi040.ogcprl (mgr.14164) 137 : cluster [DBG] pgmap v130: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:25:47.818 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:25:46 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:25:45.199785+0000 mgr.smithi040.ogcprl (mgr.14164) 138 : cluster [DBG] pgmap v131: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:25:48.739 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:25:48 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:25:47.200103+0000 mgr.smithi040.ogcprl (mgr.14164) 139 : cluster [DBG] pgmap v132: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:25:48.972 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:48.970+0000 7fc0cd72b700 1 -- 172.21.15.66:0/1817580690 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fc0c80f5410 msgr2=0x7fc0c80f5830 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:25:48.972 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:48.970+0000 7fc0cd72b700 1 --2- 172.21.15.66:0/1817580690 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fc0c80f5410 0x7fc0c80f5830 secure :-1 s=READY pgs=205 cs=0 l=1 rev1=1 rx=0x7fc0bc009fb0 tx=0x7fc0bc009b30).stop 2023-11-17T05:25:48.972 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:48.970+0000 7fc0cd72b700 1 -- 172.21.15.66:0/1817580690 shutdown_connections 2023-11-17T05:25:48.972 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:48.970+0000 7fc0cd72b700 1 --2- 172.21.15.66:0/1817580690 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fc0c80f5410 0x7fc0c80f5830 unknown :-1 s=CLOSED pgs=205 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:25:48.972 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:48.970+0000 7fc0cd72b700 1 -- 172.21.15.66:0/1817580690 >> 172.21.15.66:0/1817580690 conn(0x7fc0c80f0b70 msgr2=0x7fc0c80f2f90 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:25:48.972 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:48.970+0000 7fc0cd72b700 1 -- 172.21.15.66:0/1817580690 shutdown_connections 2023-11-17T05:25:48.972 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:48.970+0000 7fc0cd72b700 1 -- 172.21.15.66:0/1817580690 wait complete. 2023-11-17T05:25:48.972 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:48.970+0000 7fc0cd72b700 1 Processor -- start 2023-11-17T05:25:48.972 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:48.971+0000 7fc0cd72b700 1 -- start start 2023-11-17T05:25:48.972 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:48.971+0000 7fc0cd72b700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fc0c80f5410 0x7fc0c81066e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:25:48.973 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:48.971+0000 7fc0cd72b700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fc0c8106bd0 con 0x7fc0c80f5410 2023-11-17T05:25:48.973 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:48.971+0000 7fc0c6ffd700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fc0c80f5410 0x7fc0c81066e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:25:48.973 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:48.971+0000 7fc0c6ffd700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fc0c80f5410 0x7fc0c81066e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.66:40052/0 (socket says 172.21.15.66:40052) 2023-11-17T05:25:48.973 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:48.971+0000 7fc0c6ffd700 1 -- 172.21.15.66:0/1178627019 learned_addr learned my addr 172.21.15.66:0/1178627019 (peer_addr_for_me v2:172.21.15.66:0/0) 2023-11-17T05:25:48.973 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:48.971+0000 7fc0c6ffd700 1 -- 172.21.15.66:0/1178627019 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fc0bc005040 con 0x7fc0c80f5410 2023-11-17T05:25:48.973 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:48.971+0000 7fc0c6ffd700 1 --2- 172.21.15.66:0/1178627019 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fc0c80f5410 0x7fc0c81066e0 secure :-1 s=READY pgs=206 cs=0 l=1 rev1=1 rx=0x7fc0bc013510 tx=0x7fc0bc00b040).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:25:48.973 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:48.971+0000 7fc0af7fe700 1 -- 172.21.15.66:0/1178627019 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7fc0bc009320 con 0x7fc0c80f5410 2023-11-17T05:25:48.973 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:48.972+0000 7fc0af7fe700 1 -- 172.21.15.66:0/1178627019 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(25 keys) v1 ==== 1074+0+0 (secure 0 0 0) 0x7fc0bc013f00 con 0x7fc0c80f5410 2023-11-17T05:25:48.973 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:48.972+0000 7fc0cd72b700 1 -- 172.21.15.66:0/1178627019 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fc0c8106dd0 con 0x7fc0c80f5410 2023-11-17T05:25:48.973 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:48.972+0000 7fc0af7fe700 1 -- 172.21.15.66:0/1178627019 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7fc0bc014d90 con 0x7fc0c80f5410 2023-11-17T05:25:48.973 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:48.972+0000 7fc0cd72b700 1 -- 172.21.15.66:0/1178627019 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fc0c8107260 con 0x7fc0c80f5410 2023-11-17T05:25:48.977 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:48.972+0000 7fc0af7fe700 1 -- 172.21.15.66:0/1178627019 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 17) v1 ==== 36269+0+0 (secure 0 0 0) 0x7fc0bc0095f0 con 0x7fc0c80f5410 2023-11-17T05:25:48.979 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:48.972+0000 7fc0cd72b700 1 -- 172.21.15.66:0/1178627019 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fc0c8045ad0 con 0x7fc0c80f5410 2023-11-17T05:25:48.980 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:48.973+0000 7fc0af7fe700 1 --2- 172.21.15.66:0/1178627019 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fc0b002ff60 0x7fc0b0032420 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:25:48.980 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:48.973+0000 7fc0af7fe700 1 -- 172.21.15.66:0/1178627019 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(5..5 src has 1..5) v4 ==== 1270+0+0 (secure 0 0 0) 0x7fc0bc041000 con 0x7fc0c80f5410 2023-11-17T05:25:48.981 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:48.973+0000 7fc0c67fc700 1 --2- 172.21.15.66:0/1178627019 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fc0b002ff60 0x7fc0b0032420 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:25:48.982 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:48.973+0000 7fc0c67fc700 1 --2- 172.21.15.66:0/1178627019 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fc0b002ff60 0x7fc0b0032420 secure :-1 s=READY pgs=68 cs=0 l=1 rev1=1 rx=0x7fc0b80096a0 tx=0x7fc0b8006b40).ready entity=mgr.14164 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:25:48.984 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:48.978+0000 7fc0af7fe700 1 -- 172.21.15.66:0/1178627019 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+152212 (secure 0 0 0) 0x7fc0bc014420 con 0x7fc0c80f5410 2023-11-17T05:25:49.198 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:49.198+0000 7fc0cd72b700 1 -- 172.21.15.66:0/1178627019 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "mon dump", "format": "json"} v 0) v1 -- 0x7fc0c80fae60 con 0x7fc0c80f5410 2023-11-17T05:25:51.442 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:49.199+0000 7fc0af7fe700 1 -- 172.21.15.66:0/1178627019 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mon_command_ack([{"prefix": "mon dump", "format": "json"}]=0 dumped monmap epoch 1 v1) v1 ==== 95+0+690 (secure 0 0 0) 0x7fc0bc01d7f0 con 0x7fc0c80f5410 2023-11-17T05:25:51.442 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:25:49 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:25:49.199445+0000 mon.smithi040 (mon.0) 184 : audit [DBG] from='client.? 172.21.15.66:0/1178627019' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-11-17T05:25:51.442 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:25:50 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:25:49.200347+0000 mgr.smithi040.ogcprl (mgr.14164) 140 : cluster [DBG] pgmap v133: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:25:51.442 INFO:teuthology.orchestra.run.smithi066.stdout: 2023-11-17T05:25:51.442 INFO:teuthology.orchestra.run.smithi066.stdout:{"epoch":1,"fsid":"ed6470e2-8508-11ee-8dbe-212e2dc638e7","modified":"2023-11-17T05:20:13.934038Z","created":"2023-11-17T05:20:13.934038Z","min_mon_release":16,"min_mon_release_name":"pacific","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging"],"optional":[]},"mons":[{"rank":0,"name":"smithi040","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:3300","nonce":0},{"type":"v1","addr":"172.21.15.40:6789","nonce":0}]},"addr":"172.21.15.40:6789/0","public_addr":"172.21.15.40:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2023-11-17T05:25:51.444 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:49.200+0000 7fc0cd72b700 1 -- 172.21.15.66:0/1178627019 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fc0b002ff60 msgr2=0x7fc0b0032420 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:25:51.444 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:49.200+0000 7fc0cd72b700 1 --2- 172.21.15.66:0/1178627019 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fc0b002ff60 0x7fc0b0032420 secure :-1 s=READY pgs=68 cs=0 l=1 rev1=1 rx=0x7fc0b80096a0 tx=0x7fc0b8006b40).stop 2023-11-17T05:25:51.444 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:49.200+0000 7fc0cd72b700 1 -- 172.21.15.66:0/1178627019 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fc0c80f5410 msgr2=0x7fc0c81066e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:25:51.444 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:49.200+0000 7fc0cd72b700 1 --2- 172.21.15.66:0/1178627019 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fc0c80f5410 0x7fc0c81066e0 secure :-1 s=READY pgs=206 cs=0 l=1 rev1=1 rx=0x7fc0bc013510 tx=0x7fc0bc00b040).stop 2023-11-17T05:25:51.444 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:49.200+0000 7fc0cd72b700 1 -- 172.21.15.66:0/1178627019 shutdown_connections 2023-11-17T05:25:51.444 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:49.201+0000 7fc0cd72b700 1 --2- 172.21.15.66:0/1178627019 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fc0b002ff60 0x7fc0b0032420 unknown :-1 s=CLOSED pgs=68 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:25:51.444 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:49.201+0000 7fc0cd72b700 1 --2- 172.21.15.66:0/1178627019 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fc0c80f5410 0x7fc0c81066e0 unknown :-1 s=CLOSED pgs=206 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:25:51.444 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:49.201+0000 7fc0cd72b700 1 -- 172.21.15.66:0/1178627019 >> 172.21.15.66:0/1178627019 conn(0x7fc0c80f0b70 msgr2=0x7fc0c805fe00 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:25:51.444 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:49.201+0000 7fc0cd72b700 1 -- 172.21.15.66:0/1178627019 shutdown_connections 2023-11-17T05:25:51.444 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:49.201+0000 7fc0cd72b700 1 -- 172.21.15.66:0/1178627019 wait complete. 2023-11-17T05:25:51.444 INFO:teuthology.orchestra.run.smithi066.stderr:dumped monmap epoch 1 2023-11-17T05:25:52.446 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2023-11-17T05:25:52.446 DEBUG:teuthology.orchestra.run.smithi066:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.4 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ed6470e2-8508-11ee-8dbe-212e2dc638e7 -- ceph mon dump -f json 2023-11-17T05:25:52.739 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:25:52 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:25:51.200705+0000 mgr.smithi040.ogcprl (mgr.14164) 141 : cluster [DBG] pgmap v134: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:25:53.669 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:53.669+0000 7f5c3359e700 1 -- 172.21.15.66:0/4193695225 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f5c340f51a0 msgr2=0x7f5c340f55c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:25:53.669 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:53.669+0000 7f5c3359e700 1 --2- 172.21.15.66:0/4193695225 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f5c340f51a0 0x7f5c340f55c0 secure :-1 s=READY pgs=207 cs=0 l=1 rev1=1 rx=0x7f5c1c009fb0 tx=0x7f5c1c009b30).stop 2023-11-17T05:25:53.670 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:53.669+0000 7f5c3359e700 1 -- 172.21.15.66:0/4193695225 shutdown_connections 2023-11-17T05:25:53.670 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:53.669+0000 7f5c3359e700 1 --2- 172.21.15.66:0/4193695225 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f5c340f51a0 0x7f5c340f55c0 unknown :-1 s=CLOSED pgs=207 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:25:53.670 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:53.669+0000 7f5c3359e700 1 -- 172.21.15.66:0/4193695225 >> 172.21.15.66:0/4193695225 conn(0x7f5c340f09a0 msgr2=0x7f5c340f2dc0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:25:53.670 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:53.669+0000 7f5c3359e700 1 -- 172.21.15.66:0/4193695225 shutdown_connections 2023-11-17T05:25:53.670 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:53.669+0000 7f5c3359e700 1 -- 172.21.15.66:0/4193695225 wait complete. 2023-11-17T05:25:53.673 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:53.669+0000 7f5c3359e700 1 Processor -- start 2023-11-17T05:25:53.673 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:53.670+0000 7f5c3359e700 1 -- start start 2023-11-17T05:25:53.673 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:53.670+0000 7f5c3359e700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f5c340f51a0 0x7f5c341091b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:25:53.673 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:53.670+0000 7f5c3359e700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f5c34060240 con 0x7f5c340f51a0 2023-11-17T05:25:53.673 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:53.670+0000 7f5c3259c700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f5c340f51a0 0x7f5c341091b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:25:53.673 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:53.670+0000 7f5c3259c700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f5c340f51a0 0x7f5c341091b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.66:43950/0 (socket says 172.21.15.66:43950) 2023-11-17T05:25:53.673 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:53.670+0000 7f5c3259c700 1 -- 172.21.15.66:0/3546517868 learned_addr learned my addr 172.21.15.66:0/3546517868 (peer_addr_for_me v2:172.21.15.66:0/0) 2023-11-17T05:25:53.674 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:53.670+0000 7f5c3259c700 1 -- 172.21.15.66:0/3546517868 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f5c1c005040 con 0x7f5c340f51a0 2023-11-17T05:25:53.674 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:53.670+0000 7f5c3259c700 1 --2- 172.21.15.66:0/3546517868 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f5c340f51a0 0x7f5c341091b0 secure :-1 s=READY pgs=208 cs=0 l=1 rev1=1 rx=0x7f5c1c013510 tx=0x7f5c1c00b040).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:25:53.674 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:53.670+0000 7f5c2affd700 1 -- 172.21.15.66:0/3546517868 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f5c1c009320 con 0x7f5c340f51a0 2023-11-17T05:25:53.674 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:53.671+0000 7f5c3359e700 1 -- 172.21.15.66:0/3546517868 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f5c34060440 con 0x7f5c340f51a0 2023-11-17T05:25:53.674 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:53.671+0000 7f5c3359e700 1 -- 172.21.15.66:0/3546517868 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f5c34109860 con 0x7f5c340f51a0 2023-11-17T05:25:53.674 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:53.671+0000 7f5c2affd700 1 -- 172.21.15.66:0/3546517868 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(25 keys) v1 ==== 1074+0+0 (secure 0 0 0) 0x7f5c1c013f00 con 0x7f5c340f51a0 2023-11-17T05:25:53.674 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:53.671+0000 7f5c2affd700 1 -- 172.21.15.66:0/3546517868 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f5c1c014cc0 con 0x7f5c340f51a0 2023-11-17T05:25:53.674 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:53.671+0000 7f5c3359e700 1 -- 172.21.15.66:0/3546517868 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f5c18004fa0 con 0x7f5c340f51a0 2023-11-17T05:25:53.674 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:53.672+0000 7f5c2affd700 1 -- 172.21.15.66:0/3546517868 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 17) v1 ==== 36269+0+0 (secure 0 0 0) 0x7f5c1c014420 con 0x7f5c340f51a0 2023-11-17T05:25:53.674 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:53.672+0000 7f5c2affd700 1 --2- 172.21.15.66:0/3546517868 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f5c2002ff60 0x7f5c20032420 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:25:53.674 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:53.672+0000 7f5c2affd700 1 -- 172.21.15.66:0/3546517868 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(5..5 src has 1..5) v4 ==== 1270+0+0 (secure 0 0 0) 0x7f5c1c040fb0 con 0x7f5c340f51a0 2023-11-17T05:25:53.674 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:53.672+0000 7f5c31d9b700 1 --2- 172.21.15.66:0/3546517868 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f5c2002ff60 0x7f5c20032420 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:25:53.675 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:53.672+0000 7f5c31d9b700 1 --2- 172.21.15.66:0/3546517868 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f5c2002ff60 0x7f5c20032420 secure :-1 s=READY pgs=69 cs=0 l=1 rev1=1 rx=0x7f5c240096a0 tx=0x7f5c24006b40).ready entity=mgr.14164 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:25:53.677 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:53.677+0000 7f5c2affd700 1 -- 172.21.15.66:0/3546517868 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+152212 (secure 0 0 0) 0x7f5c1c0095f0 con 0x7f5c340f51a0 2023-11-17T05:25:53.897 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:53.896+0000 7f5c3359e700 1 -- 172.21.15.66:0/3546517868 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "mon dump", "format": "json"} v 0) v1 -- 0x7f5c18005670 con 0x7f5c340f51a0 2023-11-17T05:25:53.897 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:53.897+0000 7f5c2affd700 1 -- 172.21.15.66:0/3546517868 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mon_command_ack([{"prefix": "mon dump", "format": "json"}]=0 dumped monmap epoch 1 v1) v1 ==== 95+0+690 (secure 0 0 0) 0x7f5c1c01db70 con 0x7f5c340f51a0 2023-11-17T05:25:53.897 INFO:teuthology.orchestra.run.smithi066.stdout: 2023-11-17T05:25:53.898 INFO:teuthology.orchestra.run.smithi066.stdout:{"epoch":1,"fsid":"ed6470e2-8508-11ee-8dbe-212e2dc638e7","modified":"2023-11-17T05:20:13.934038Z","created":"2023-11-17T05:20:13.934038Z","min_mon_release":16,"min_mon_release_name":"pacific","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging"],"optional":[]},"mons":[{"rank":0,"name":"smithi040","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:3300","nonce":0},{"type":"v1","addr":"172.21.15.40:6789","nonce":0}]},"addr":"172.21.15.40:6789/0","public_addr":"172.21.15.40:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2023-11-17T05:25:53.899 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:53.899+0000 7f5c3359e700 1 -- 172.21.15.66:0/3546517868 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f5c2002ff60 msgr2=0x7f5c20032420 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:25:53.899 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:53.899+0000 7f5c3359e700 1 --2- 172.21.15.66:0/3546517868 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f5c2002ff60 0x7f5c20032420 secure :-1 s=READY pgs=69 cs=0 l=1 rev1=1 rx=0x7f5c240096a0 tx=0x7f5c24006b40).stop 2023-11-17T05:25:53.899 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:53.899+0000 7f5c3359e700 1 -- 172.21.15.66:0/3546517868 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f5c340f51a0 msgr2=0x7f5c341091b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:25:53.899 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:53.899+0000 7f5c3359e700 1 --2- 172.21.15.66:0/3546517868 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f5c340f51a0 0x7f5c341091b0 secure :-1 s=READY pgs=208 cs=0 l=1 rev1=1 rx=0x7f5c1c013510 tx=0x7f5c1c00b040).stop 2023-11-17T05:25:53.899 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:53.899+0000 7f5c3359e700 1 -- 172.21.15.66:0/3546517868 shutdown_connections 2023-11-17T05:25:53.899 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:53.899+0000 7f5c3359e700 1 --2- 172.21.15.66:0/3546517868 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f5c2002ff60 0x7f5c20032420 unknown :-1 s=CLOSED pgs=69 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:25:53.899 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:53.899+0000 7f5c3359e700 1 --2- 172.21.15.66:0/3546517868 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f5c340f51a0 0x7f5c341091b0 unknown :-1 s=CLOSED pgs=208 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:25:53.900 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:53.899+0000 7f5c3359e700 1 -- 172.21.15.66:0/3546517868 >> 172.21.15.66:0/3546517868 conn(0x7f5c340f09a0 msgr2=0x7f5c340f1460 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:25:53.900 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:53.899+0000 7f5c3359e700 1 -- 172.21.15.66:0/3546517868 shutdown_connections 2023-11-17T05:25:53.900 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:53.899+0000 7f5c3359e700 1 -- 172.21.15.66:0/3546517868 wait complete. 2023-11-17T05:25:53.900 INFO:teuthology.orchestra.run.smithi066.stderr:dumped monmap epoch 1 2023-11-17T05:25:54.739 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:25:54 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:25:53.200984+0000 mgr.smithi040.ogcprl (mgr.14164) 142 : cluster [DBG] pgmap v135: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:25:54.739 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:25:54 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:25:53.897619+0000 mon.smithi040 (mon.0) 185 : audit [DBG] from='client.? 172.21.15.66:0/3546517868' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-11-17T05:25:55.304 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2023-11-17T05:25:55.304 DEBUG:teuthology.orchestra.run.smithi066:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.4 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ed6470e2-8508-11ee-8dbe-212e2dc638e7 -- ceph mon dump -f json 2023-11-17T05:25:56.548 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:56.539+0000 7efc0b0f0700 1 -- 172.21.15.66:0/2376207957 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7efc04060280 msgr2=0x7efc040f90f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:25:56.548 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:56.539+0000 7efc0b0f0700 1 --2- 172.21.15.66:0/2376207957 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7efc04060280 0x7efc040f90f0 secure :-1 s=READY pgs=209 cs=0 l=1 rev1=1 rx=0x7efbf8009fb0 tx=0x7efbf8009b30).stop 2023-11-17T05:25:56.548 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:56.539+0000 7efc0b0f0700 1 -- 172.21.15.66:0/2376207957 shutdown_connections 2023-11-17T05:25:56.548 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:56.539+0000 7efc0b0f0700 1 --2- 172.21.15.66:0/2376207957 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7efc04060280 0x7efc040f90f0 unknown :-1 s=CLOSED pgs=209 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:25:56.549 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:56.539+0000 7efc0b0f0700 1 -- 172.21.15.66:0/2376207957 >> 172.21.15.66:0/2376207957 conn(0x7efc040f0b70 msgr2=0x7efc040f2f90 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:25:56.549 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:56.539+0000 7efc0b0f0700 1 -- 172.21.15.66:0/2376207957 shutdown_connections 2023-11-17T05:25:56.549 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:56.539+0000 7efc0b0f0700 1 -- 172.21.15.66:0/2376207957 wait complete. 2023-11-17T05:25:56.549 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:56.540+0000 7efc0b0f0700 1 Processor -- start 2023-11-17T05:25:56.549 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:56.540+0000 7efc0b0f0700 1 -- start start 2023-11-17T05:25:56.549 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:56.540+0000 7efc0b0f0700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7efc04060280 0x7efc041066e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:25:56.549 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:56.540+0000 7efc0b0f0700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7efc04106bd0 con 0x7efc04060280 2023-11-17T05:25:56.549 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:56.541+0000 7efc08e8c700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7efc04060280 0x7efc041066e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:25:56.549 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:56.541+0000 7efc08e8c700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7efc04060280 0x7efc041066e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.66:43960/0 (socket says 172.21.15.66:43960) 2023-11-17T05:25:56.549 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:56.541+0000 7efc08e8c700 1 -- 172.21.15.66:0/897121446 learned_addr learned my addr 172.21.15.66:0/897121446 (peer_addr_for_me v2:172.21.15.66:0/0) 2023-11-17T05:25:56.549 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:56.541+0000 7efc08e8c700 1 -- 172.21.15.66:0/897121446 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7efbf8005040 con 0x7efc04060280 2023-11-17T05:25:56.550 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:56.541+0000 7efc08e8c700 1 --2- 172.21.15.66:0/897121446 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7efc04060280 0x7efc041066e0 secure :-1 s=READY pgs=210 cs=0 l=1 rev1=1 rx=0x7efbf8013510 tx=0x7efbf800b040).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:25:56.550 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:56.541+0000 7efc017fa700 1 -- 172.21.15.66:0/897121446 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7efbf8009320 con 0x7efc04060280 2023-11-17T05:25:56.550 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:56.541+0000 7efc017fa700 1 -- 172.21.15.66:0/897121446 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(25 keys) v1 ==== 1074+0+0 (secure 0 0 0) 0x7efbf8013f00 con 0x7efc04060280 2023-11-17T05:25:56.550 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:56.541+0000 7efc0b0f0700 1 -- 172.21.15.66:0/897121446 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7efc04106dd0 con 0x7efc04060280 2023-11-17T05:25:56.550 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:56.541+0000 7efc017fa700 1 -- 172.21.15.66:0/897121446 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7efbf8014dc0 con 0x7efc04060280 2023-11-17T05:25:56.550 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:56.541+0000 7efc0b0f0700 1 -- 172.21.15.66:0/897121446 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7efc04107260 con 0x7efc04060280 2023-11-17T05:25:56.550 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:56.542+0000 7efc017fa700 1 -- 172.21.15.66:0/897121446 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 17) v1 ==== 36269+0+0 (secure 0 0 0) 0x7efbf8014420 con 0x7efc04060280 2023-11-17T05:25:56.550 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:56.542+0000 7efc0b0f0700 1 -- 172.21.15.66:0/897121446 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7efc04045ad0 con 0x7efc04060280 2023-11-17T05:25:56.550 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:56.542+0000 7efc017fa700 1 --2- 172.21.15.66:0/897121446 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7efbf402ff60 0x7efbf4032420 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:25:56.550 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:56.542+0000 7efc017fa700 1 -- 172.21.15.66:0/897121446 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(5..5 src has 1..5) v4 ==== 1270+0+0 (secure 0 0 0) 0x7efbf8041110 con 0x7efc04060280 2023-11-17T05:25:56.550 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:56.543+0000 7efc03fff700 1 --2- 172.21.15.66:0/897121446 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7efbf402ff60 0x7efbf4032420 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:25:56.550 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:56.543+0000 7efc03fff700 1 --2- 172.21.15.66:0/897121446 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7efbf402ff60 0x7efbf4032420 secure :-1 s=READY pgs=70 cs=0 l=1 rev1=1 rx=0x7efbf00096a0 tx=0x7efbf0006b40).ready entity=mgr.14164 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:25:56.551 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:56.547+0000 7efc017fa700 1 -- 172.21.15.66:0/897121446 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+152212 (secure 0 0 0) 0x7efbf80095f0 con 0x7efc04060280 2023-11-17T05:25:56.739 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:25:56 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:25:55.201404+0000 mgr.smithi040.ogcprl (mgr.14164) 143 : cluster [DBG] pgmap v136: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:25:56.765 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:56.765+0000 7efc0b0f0700 1 -- 172.21.15.66:0/897121446 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "mon dump", "format": "json"} v 0) v1 -- 0x7efc04108110 con 0x7efc04060280 2023-11-17T05:25:56.766 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:56.765+0000 7efc017fa700 1 -- 172.21.15.66:0/897121446 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mon_command_ack([{"prefix": "mon dump", "format": "json"}]=0 dumped monmap epoch 1 v1) v1 ==== 95+0+690 (secure 0 0 0) 0x7efbf801cb10 con 0x7efc04060280 2023-11-17T05:25:56.766 INFO:teuthology.orchestra.run.smithi066.stdout: 2023-11-17T05:25:56.766 INFO:teuthology.orchestra.run.smithi066.stdout:{"epoch":1,"fsid":"ed6470e2-8508-11ee-8dbe-212e2dc638e7","modified":"2023-11-17T05:20:13.934038Z","created":"2023-11-17T05:20:13.934038Z","min_mon_release":16,"min_mon_release_name":"pacific","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging"],"optional":[]},"mons":[{"rank":0,"name":"smithi040","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:3300","nonce":0},{"type":"v1","addr":"172.21.15.40:6789","nonce":0}]},"addr":"172.21.15.40:6789/0","public_addr":"172.21.15.40:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2023-11-17T05:25:56.768 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:56.767+0000 7efc0b0f0700 1 -- 172.21.15.66:0/897121446 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7efbf402ff60 msgr2=0x7efbf4032420 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:25:56.768 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:56.767+0000 7efc0b0f0700 1 --2- 172.21.15.66:0/897121446 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7efbf402ff60 0x7efbf4032420 secure :-1 s=READY pgs=70 cs=0 l=1 rev1=1 rx=0x7efbf00096a0 tx=0x7efbf0006b40).stop 2023-11-17T05:25:56.768 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:56.767+0000 7efc0b0f0700 1 -- 172.21.15.66:0/897121446 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7efc04060280 msgr2=0x7efc041066e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:25:56.768 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:56.767+0000 7efc0b0f0700 1 --2- 172.21.15.66:0/897121446 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7efc04060280 0x7efc041066e0 secure :-1 s=READY pgs=210 cs=0 l=1 rev1=1 rx=0x7efbf8013510 tx=0x7efbf800b040).stop 2023-11-17T05:25:56.768 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:56.767+0000 7efc0b0f0700 1 -- 172.21.15.66:0/897121446 shutdown_connections 2023-11-17T05:25:56.768 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:56.767+0000 7efc0b0f0700 1 --2- 172.21.15.66:0/897121446 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7efbf402ff60 0x7efbf4032420 unknown :-1 s=CLOSED pgs=70 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:25:56.768 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:56.767+0000 7efc0b0f0700 1 --2- 172.21.15.66:0/897121446 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7efc04060280 0x7efc041066e0 unknown :-1 s=CLOSED pgs=210 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:25:56.768 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:56.767+0000 7efc0b0f0700 1 -- 172.21.15.66:0/897121446 >> 172.21.15.66:0/897121446 conn(0x7efc040f0b70 msgr2=0x7efc040f1660 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:25:56.768 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:56.767+0000 7efc0b0f0700 1 -- 172.21.15.66:0/897121446 shutdown_connections 2023-11-17T05:25:56.768 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:56.767+0000 7efc0b0f0700 1 -- 172.21.15.66:0/897121446 wait complete. 2023-11-17T05:25:56.769 INFO:teuthology.orchestra.run.smithi066.stderr:dumped monmap epoch 1 2023-11-17T05:25:57.739 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:25:57 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:25:56.766106+0000 mon.smithi040 (mon.0) 186 : audit [DBG] from='client.? 172.21.15.66:0/897121446' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-11-17T05:25:58.191 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2023-11-17T05:25:58.191 DEBUG:teuthology.orchestra.run.smithi066:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.4 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ed6470e2-8508-11ee-8dbe-212e2dc638e7 -- ceph mon dump -f json 2023-11-17T05:25:58.739 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:25:58 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:25:57.201748+0000 mgr.smithi040.ogcprl (mgr.14164) 144 : cluster [DBG] pgmap v137: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:25:59.302 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:59.301+0000 7fb3478fd700 1 -- 172.21.15.66:0/4014120407 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fb340060280 msgr2=0x7fb3400f90f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:25:59.303 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:59.301+0000 7fb3478fd700 1 --2- 172.21.15.66:0/4014120407 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fb340060280 0x7fb3400f90f0 secure :-1 s=READY pgs=211 cs=0 l=1 rev1=1 rx=0x7fb33c009fb0 tx=0x7fb33c009b30).stop 2023-11-17T05:25:59.303 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:59.301+0000 7fb3478fd700 1 -- 172.21.15.66:0/4014120407 shutdown_connections 2023-11-17T05:25:59.303 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:59.301+0000 7fb3478fd700 1 --2- 172.21.15.66:0/4014120407 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fb340060280 0x7fb3400f90f0 unknown :-1 s=CLOSED pgs=211 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:25:59.303 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:59.301+0000 7fb3478fd700 1 -- 172.21.15.66:0/4014120407 >> 172.21.15.66:0/4014120407 conn(0x7fb3400f0b70 msgr2=0x7fb3400f2f90 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:25:59.303 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:59.301+0000 7fb3478fd700 1 -- 172.21.15.66:0/4014120407 shutdown_connections 2023-11-17T05:25:59.303 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:59.301+0000 7fb3478fd700 1 -- 172.21.15.66:0/4014120407 wait complete. 2023-11-17T05:25:59.303 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:59.302+0000 7fb3478fd700 1 Processor -- start 2023-11-17T05:25:59.303 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:59.302+0000 7fb3478fd700 1 -- start start 2023-11-17T05:25:59.303 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:59.302+0000 7fb3478fd700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fb340060280 0x7fb3400f6b70 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:25:59.303 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:59.302+0000 7fb3478fd700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fb3400f4ff0 con 0x7fb340060280 2023-11-17T05:25:59.303 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:59.302+0000 7fb345699700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fb340060280 0x7fb3400f6b70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:25:59.304 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:59.302+0000 7fb345699700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fb340060280 0x7fb3400f6b70 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.66:43982/0 (socket says 172.21.15.66:43982) 2023-11-17T05:25:59.304 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:59.302+0000 7fb345699700 1 -- 172.21.15.66:0/3089094301 learned_addr learned my addr 172.21.15.66:0/3089094301 (peer_addr_for_me v2:172.21.15.66:0/0) 2023-11-17T05:25:59.305 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:59.303+0000 7fb345699700 1 -- 172.21.15.66:0/3089094301 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fb33c005040 con 0x7fb340060280 2023-11-17T05:25:59.305 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:59.303+0000 7fb345699700 1 --2- 172.21.15.66:0/3089094301 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fb340060280 0x7fb3400f6b70 secure :-1 s=READY pgs=212 cs=0 l=1 rev1=1 rx=0x7fb33c013510 tx=0x7fb33c00b040).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:25:59.305 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:59.303+0000 7fb335ffb700 1 -- 172.21.15.66:0/3089094301 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7fb33c009320 con 0x7fb340060280 2023-11-17T05:25:59.305 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:59.303+0000 7fb335ffb700 1 -- 172.21.15.66:0/3089094301 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(25 keys) v1 ==== 1074+0+0 (secure 0 0 0) 0x7fb33c013f00 con 0x7fb340060280 2023-11-17T05:25:59.305 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:59.303+0000 7fb3478fd700 1 -- 172.21.15.66:0/3089094301 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fb3400f5190 con 0x7fb340060280 2023-11-17T05:25:59.305 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:59.303+0000 7fb335ffb700 1 -- 172.21.15.66:0/3089094301 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7fb33c014dc0 con 0x7fb340060280 2023-11-17T05:25:59.305 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:59.303+0000 7fb3478fd700 1 -- 172.21.15.66:0/3089094301 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fb3400f5620 con 0x7fb340060280 2023-11-17T05:25:59.306 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:59.304+0000 7fb335ffb700 1 -- 172.21.15.66:0/3089094301 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 17) v1 ==== 36269+0+0 (secure 0 0 0) 0x7fb33c01d3d0 con 0x7fb340060280 2023-11-17T05:25:59.306 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:59.304+0000 7fb335ffb700 1 --2- 172.21.15.66:0/3089094301 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fb32c02ff60 0x7fb32c032420 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:25:59.306 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:59.304+0000 7fb335ffb700 1 -- 172.21.15.66:0/3089094301 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(5..5 src has 1..5) v4 ==== 1270+0+0 (secure 0 0 0) 0x7fb33c041050 con 0x7fb340060280 2023-11-17T05:25:59.306 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:59.304+0000 7fb3478fd700 1 -- 172.21.15.66:0/3089094301 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fb340059100 con 0x7fb340060280 2023-11-17T05:25:59.306 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:59.304+0000 7fb344e98700 1 --2- 172.21.15.66:0/3089094301 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fb32c02ff60 0x7fb32c032420 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:25:59.306 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:59.305+0000 7fb344e98700 1 --2- 172.21.15.66:0/3089094301 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fb32c02ff60 0x7fb32c032420 secure :-1 s=READY pgs=71 cs=0 l=1 rev1=1 rx=0x7fb3300096a0 tx=0x7fb330006b40).ready entity=mgr.14164 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:25:59.309 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:59.309+0000 7fb335ffb700 1 -- 172.21.15.66:0/3089094301 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+152212 (secure 0 0 0) 0x7fb33c012920 con 0x7fb340060280 2023-11-17T05:25:59.530 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:59.530+0000 7fb3478fd700 1 -- 172.21.15.66:0/3089094301 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "mon dump", "format": "json"} v 0) v1 -- 0x7fb3400fbf50 con 0x7fb340060280 2023-11-17T05:25:59.531 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:59.531+0000 7fb335ffb700 1 -- 172.21.15.66:0/3089094301 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mon_command_ack([{"prefix": "mon dump", "format": "json"}]=0 dumped monmap epoch 1 v1) v1 ==== 95+0+690 (secure 0 0 0) 0x7fb33c02b020 con 0x7fb340060280 2023-11-17T05:25:59.531 INFO:teuthology.orchestra.run.smithi066.stdout: 2023-11-17T05:25:59.531 INFO:teuthology.orchestra.run.smithi066.stdout:{"epoch":1,"fsid":"ed6470e2-8508-11ee-8dbe-212e2dc638e7","modified":"2023-11-17T05:20:13.934038Z","created":"2023-11-17T05:20:13.934038Z","min_mon_release":16,"min_mon_release_name":"pacific","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging"],"optional":[]},"mons":[{"rank":0,"name":"smithi040","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:3300","nonce":0},{"type":"v1","addr":"172.21.15.40:6789","nonce":0}]},"addr":"172.21.15.40:6789/0","public_addr":"172.21.15.40:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2023-11-17T05:25:59.533 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:59.532+0000 7fb3478fd700 1 -- 172.21.15.66:0/3089094301 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fb32c02ff60 msgr2=0x7fb32c032420 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:25:59.533 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:59.532+0000 7fb3478fd700 1 --2- 172.21.15.66:0/3089094301 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fb32c02ff60 0x7fb32c032420 secure :-1 s=READY pgs=71 cs=0 l=1 rev1=1 rx=0x7fb3300096a0 tx=0x7fb330006b40).stop 2023-11-17T05:25:59.533 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:59.532+0000 7fb3478fd700 1 -- 172.21.15.66:0/3089094301 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fb340060280 msgr2=0x7fb3400f6b70 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:25:59.533 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:59.532+0000 7fb3478fd700 1 --2- 172.21.15.66:0/3089094301 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fb340060280 0x7fb3400f6b70 secure :-1 s=READY pgs=212 cs=0 l=1 rev1=1 rx=0x7fb33c013510 tx=0x7fb33c00b040).stop 2023-11-17T05:25:59.533 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:59.532+0000 7fb3478fd700 1 -- 172.21.15.66:0/3089094301 shutdown_connections 2023-11-17T05:25:59.533 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:59.532+0000 7fb3478fd700 1 --2- 172.21.15.66:0/3089094301 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fb32c02ff60 0x7fb32c032420 unknown :-1 s=CLOSED pgs=71 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:25:59.533 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:59.532+0000 7fb3478fd700 1 --2- 172.21.15.66:0/3089094301 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fb340060280 0x7fb3400f6b70 unknown :-1 s=CLOSED pgs=212 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:25:59.533 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:59.532+0000 7fb3478fd700 1 -- 172.21.15.66:0/3089094301 >> 172.21.15.66:0/3089094301 conn(0x7fb3400f0b70 msgr2=0x7fb3400f1660 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:25:59.533 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:59.532+0000 7fb3478fd700 1 -- 172.21.15.66:0/3089094301 shutdown_connections 2023-11-17T05:25:59.533 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:25:59.532+0000 7fb3478fd700 1 -- 172.21.15.66:0/3089094301 wait complete. 2023-11-17T05:25:59.534 INFO:teuthology.orchestra.run.smithi066.stderr:dumped monmap epoch 1 2023-11-17T05:26:00.739 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:26:00 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:25:59.202014+0000 mgr.smithi040.ogcprl (mgr.14164) 145 : cluster [DBG] pgmap v138: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:26:00.739 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:26:00 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:25:59.531384+0000 mon.smithi040 (mon.0) 187 : audit [DBG] from='client.? 172.21.15.66:0/3089094301' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-11-17T05:26:00.905 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2023-11-17T05:26:00.905 DEBUG:teuthology.orchestra.run.smithi066:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.4 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ed6470e2-8508-11ee-8dbe-212e2dc638e7 -- ceph mon dump -f json 2023-11-17T05:26:02.084 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:26:02.083+0000 7f98b80de700 1 -- 172.21.15.66:0/2854879381 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f98b00f5410 msgr2=0x7f98b00f5830 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:26:02.084 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:26:02.083+0000 7f98b80de700 1 --2- 172.21.15.66:0/2854879381 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f98b00f5410 0x7f98b00f5830 secure :-1 s=READY pgs=213 cs=0 l=1 rev1=1 rx=0x7f98ac009fb0 tx=0x7f98ac009b30).stop 2023-11-17T05:26:02.085 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:26:02.083+0000 7f98b80de700 1 -- 172.21.15.66:0/2854879381 shutdown_connections 2023-11-17T05:26:02.085 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:26:02.083+0000 7f98b80de700 1 --2- 172.21.15.66:0/2854879381 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f98b00f5410 0x7f98b00f5830 unknown :-1 s=CLOSED pgs=213 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:26:02.085 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:26:02.083+0000 7f98b80de700 1 -- 172.21.15.66:0/2854879381 >> 172.21.15.66:0/2854879381 conn(0x7f98b00f0b70 msgr2=0x7f98b00f2f90 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:26:02.085 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:26:02.083+0000 7f98b80de700 1 -- 172.21.15.66:0/2854879381 shutdown_connections 2023-11-17T05:26:02.085 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:26:02.083+0000 7f98b80de700 1 -- 172.21.15.66:0/2854879381 wait complete. 2023-11-17T05:26:02.085 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:26:02.084+0000 7f98b80de700 1 Processor -- start 2023-11-17T05:26:02.085 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:26:02.084+0000 7f98b80de700 1 -- start start 2023-11-17T05:26:02.085 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:26:02.084+0000 7f98b80de700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f98b00f5410 0x7f98b0106710 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:26:02.085 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:26:02.084+0000 7f98b80de700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f98b0106c00 con 0x7f98b00f5410 2023-11-17T05:26:02.085 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:26:02.084+0000 7f98b5e7a700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f98b00f5410 0x7f98b0106710 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:26:02.085 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:26:02.084+0000 7f98b5e7a700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f98b00f5410 0x7f98b0106710 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.66:43998/0 (socket says 172.21.15.66:43998) 2023-11-17T05:26:02.085 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:26:02.084+0000 7f98b5e7a700 1 -- 172.21.15.66:0/2618265248 learned_addr learned my addr 172.21.15.66:0/2618265248 (peer_addr_for_me v2:172.21.15.66:0/0) 2023-11-17T05:26:02.085 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:26:02.084+0000 7f98b5e7a700 1 -- 172.21.15.66:0/2618265248 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f98ac005040 con 0x7f98b00f5410 2023-11-17T05:26:02.086 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:26:02.084+0000 7f98b5e7a700 1 --2- 172.21.15.66:0/2618265248 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f98b00f5410 0x7f98b0106710 secure :-1 s=READY pgs=214 cs=0 l=1 rev1=1 rx=0x7f98ac013510 tx=0x7f98ac00b040).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:26:02.087 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:26:02.084+0000 7f98a27fc700 1 -- 172.21.15.66:0/2618265248 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f98ac009320 con 0x7f98b00f5410 2023-11-17T05:26:02.087 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:26:02.085+0000 7f98b80de700 1 -- 172.21.15.66:0/2618265248 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f98b0106e00 con 0x7f98b00f5410 2023-11-17T05:26:02.087 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:26:02.085+0000 7f98a27fc700 1 -- 172.21.15.66:0/2618265248 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(25 keys) v1 ==== 1074+0+0 (secure 0 0 0) 0x7f98ac013f00 con 0x7f98b00f5410 2023-11-17T05:26:02.087 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:26:02.085+0000 7f98a27fc700 1 -- 172.21.15.66:0/2618265248 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f98ac014d90 con 0x7f98b00f5410 2023-11-17T05:26:02.087 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:26:02.085+0000 7f98b80de700 1 -- 172.21.15.66:0/2618265248 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f98b0107290 con 0x7f98b00f5410 2023-11-17T05:26:02.087 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:26:02.085+0000 7f98a27fc700 1 -- 172.21.15.66:0/2618265248 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 17) v1 ==== 36269+0+0 (secure 0 0 0) 0x7f98ac0095f0 con 0x7f98b00f5410 2023-11-17T05:26:02.087 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:26:02.085+0000 7f98b80de700 1 -- 172.21.15.66:0/2618265248 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f98b0045ad0 con 0x7f98b00f5410 2023-11-17T05:26:02.087 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:26:02.086+0000 7f98a27fc700 1 --2- 172.21.15.66:0/2618265248 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f989c02ffb0 0x7f989c032470 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:26:02.087 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:26:02.086+0000 7f98a27fc700 1 -- 172.21.15.66:0/2618265248 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(5..5 src has 1..5) v4 ==== 1270+0+0 (secure 0 0 0) 0x7f98ac041050 con 0x7f98b00f5410 2023-11-17T05:26:02.087 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:26:02.086+0000 7f98b5679700 1 --2- 172.21.15.66:0/2618265248 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f989c02ffb0 0x7f989c032470 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:26:02.087 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:26:02.086+0000 7f98b5679700 1 --2- 172.21.15.66:0/2618265248 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f989c02ffb0 0x7f989c032470 secure :-1 s=READY pgs=72 cs=0 l=1 rev1=1 rx=0x7f98a40096a0 tx=0x7f98a4006b40).ready entity=mgr.14164 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:27:23.602 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:26:02.091+0000 7f98a27fc700 1 -- 172.21.15.66:0/2618265248 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+152212 (secure 0 0 0) 0x7f98ac014420 con 0x7f98b00f5410 2023-11-17T05:27:23.633 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:26:02.307+0000 7f98b80de700 1 -- 172.21.15.66:0/2618265248 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "mon dump", "format": "json"} v 0) v1 -- 0x7f98b00fae60 con 0x7f98b00f5410 2023-11-17T05:27:23.644 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:26:02.308+0000 7f98a27fc700 1 -- 172.21.15.66:0/2618265248 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mon_command_ack([{"prefix": "mon dump", "format": "json"}]=0 dumped monmap epoch 1 v1) v1 ==== 95+0+690 (secure 0 0 0) 0x7f98ac010230 con 0x7f98b00f5410 2023-11-17T05:27:23.662 INFO:teuthology.orchestra.run.smithi066.stdout: 2023-11-17T05:27:23.669 INFO:teuthology.orchestra.run.smithi066.stdout:{"epoch":1,"fsid":"ed6470e2-8508-11ee-8dbe-212e2dc638e7","modified":"2023-11-17T05:20:13.934038Z","created":"2023-11-17T05:20:13.934038Z","min_mon_release":16,"min_mon_release_name":"pacific","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging"],"optional":[]},"mons":[{"rank":0,"name":"smithi040","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:3300","nonce":0},{"type":"v1","addr":"172.21.15.40:6789","nonce":0}]},"addr":"172.21.15.40:6789/0","public_addr":"172.21.15.40:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2023-11-17T05:27:23.674 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:26:02.309+0000 7f98b80de700 1 -- 172.21.15.66:0/2618265248 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f989c02ffb0 msgr2=0x7f989c032470 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:27:23.711 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:26:02.309+0000 7f98b80de700 1 --2- 172.21.15.66:0/2618265248 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f989c02ffb0 0x7f989c032470 secure :-1 s=READY pgs=72 cs=0 l=1 rev1=1 rx=0x7f98a40096a0 tx=0x7f98a4006b40).stop 2023-11-17T05:27:23.711 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:26:02.309+0000 7f98b80de700 1 -- 172.21.15.66:0/2618265248 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f98b00f5410 msgr2=0x7f98b0106710 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:27:23.711 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:26:02.309+0000 7f98b80de700 1 --2- 172.21.15.66:0/2618265248 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f98b00f5410 0x7f98b0106710 secure :-1 s=READY pgs=214 cs=0 l=1 rev1=1 rx=0x7f98ac013510 tx=0x7f98ac00b040).stop 2023-11-17T05:27:23.711 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:26:02.309+0000 7f98b80de700 1 -- 172.21.15.66:0/2618265248 shutdown_connections 2023-11-17T05:27:23.711 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:26:02.309+0000 7f98b80de700 1 --2- 172.21.15.66:0/2618265248 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f989c02ffb0 0x7f989c032470 unknown :-1 s=CLOSED pgs=72 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:27:23.711 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:26:02.309+0000 7f98b80de700 1 --2- 172.21.15.66:0/2618265248 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f98b00f5410 0x7f98b0106710 unknown :-1 s=CLOSED pgs=214 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:27:23.711 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:26:02.309+0000 7f98b80de700 1 -- 172.21.15.66:0/2618265248 >> 172.21.15.66:0/2618265248 conn(0x7f98b00f0b70 msgr2=0x7f98b005fe00 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:27:23.711 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:26:02.309+0000 7f98b80de700 1 -- 172.21.15.66:0/2618265248 shutdown_connections 2023-11-17T05:27:23.711 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:26:02.309+0000 7f98b80de700 1 -- 172.21.15.66:0/2618265248 wait complete. 2023-11-17T05:27:23.711 INFO:teuthology.orchestra.run.smithi066.stderr:dumped monmap epoch 1 2023-11-17T05:27:23.715 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:26:02 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:26:01.202325+0000 mgr.smithi040.ogcprl (mgr.14164) 146 : cluster [DBG] pgmap v139: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:27:23.715 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:26:02 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:26:02.308397+0000 mon.smithi040 (mon.0) 188 : audit [DBG] from='client.? 172.21.15.66:0/2618265248' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-11-17T05:27:23.715 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:26:04 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:26:03.202586+0000 mgr.smithi040.ogcprl (mgr.14164) 147 : cluster [DBG] pgmap v140: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:27:23.715 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:26:06 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:26:05.202859+0000 mgr.smithi040.ogcprl (mgr.14164) 148 : cluster [DBG] pgmap v141: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:27:23.776 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:26:08 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:26:07.203201+0000 mgr.smithi040.ogcprl (mgr.14164) 149 : cluster [DBG] pgmap v142: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:27:23.776 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:26:10 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:26:09.203504+0000 mgr.smithi040.ogcprl (mgr.14164) 150 : cluster [DBG] pgmap v143: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:27:23.776 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:26:11 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:26:11.312322+0000 mon.smithi040 (mon.0) 189 : audit [INF] from='mgr.14164 172.21.15.40:0/2203032451' entity='mgr.smithi040.ogcprl' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi040.ogcprl/mirror_snapshot_schedule"}]: dispatch 2023-11-17T05:27:23.777 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:26:11 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:26:11.313873+0000 mon.smithi040 (mon.0) 190 : audit [INF] from='mgr.14164 172.21.15.40:0/2203032451' entity='mgr.smithi040.ogcprl' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi040.ogcprl/trash_purge_schedule"}]: dispatch 2023-11-17T05:27:23.777 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:26:12 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:26:11.203729+0000 mgr.smithi040.ogcprl (mgr.14164) 151 : cluster [DBG] pgmap v144: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:27:23.777 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:26:13 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:26:13.203950+0000 mgr.smithi040.ogcprl (mgr.14164) 152 : cluster [DBG] pgmap v145: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:27:23.777 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:26:16 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:26:15.204171+0000 mgr.smithi040.ogcprl (mgr.14164) 153 : cluster [DBG] pgmap v146: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:27:23.777 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:26:18 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:26:17.204394+0000 mgr.smithi040.ogcprl (mgr.14164) 154 : cluster [DBG] pgmap v147: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:27:23.777 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:26:20 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:26:19.204662+0000 mgr.smithi040.ogcprl (mgr.14164) 155 : cluster [DBG] pgmap v148: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:27:23.777 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:26:21 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:26:20.734725+0000 mon.smithi040 (mon.0) 191 : audit [INF] from='mgr.14164 172.21.15.40:0/2203032451' entity='mgr.smithi040.ogcprl' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi040", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2023-11-17T05:27:23.777 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:26:21 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:26:20.737911+0000 mon.smithi040 (mon.0) 192 : audit [INF] from='mgr.14164 172.21.15.40:0/2203032451' entity='mgr.smithi040.ogcprl' cmd='[{"prefix": "auth get-or-create", "entity": "client.crash.smithi040", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]': finished 2023-11-17T05:27:23.777 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:26:21 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:26:20.738752+0000 mon.smithi040 (mon.0) 193 : audit [DBG] from='mgr.14164 172.21.15.40:0/2203032451' entity='mgr.smithi040.ogcprl' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-11-17T05:27:23.777 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:26:22 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cephadm 2023-11-17T05:26:20.732990+0000 mgr.smithi040.ogcprl (mgr.14164) 156 : cephadm [ERR] cephadm exited with an error code: 1, stderr:Deploy daemon alertmanager.smithi040 ... 2023-11-17T05:27:23.777 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:26:22 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: Non-zero exit code 125 from /bin/podman run --rm --ipc=host --no-hosts --net=host --entrypoint stat --init -e CONTAINER_IMAGE=docker.io/prom/alertmanager:v0.20.0 -e NODE_NAME=smithi040 -e CEPH_USE_RANDOM_NONCE=1 docker.io/prom/alertmanager:v0.20.0 -c %u %g /etc/alertmanager 2023-11-17T05:27:23.777 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:26:22 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: stat: stderr Trying to pull docker.io/prom/alertmanager:v0.20.0... 2023-11-17T05:27:23.777 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:26:22 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: stat: stderr time="2023-11-17T05:21:59Z" level=warning msg="Failed, retrying in 1s ... (1/3). Error: initializing source docker://prom/alertmanager:v0.20.0: (Mirrors also failed: [docker-mirror.front.sepia.ceph.com:5000/prom/alertmanager:v0.20.0: pinging container registry docker-mirror.front.sepia.ceph.com:5000: Get \"https://docker-mirror.front.sepia.ceph.com:5000/v2/\": dial tcp 172.21.0.79:5000: connect: no route to host]): docker.io/prom/alertmanager:v0.20.0: reading manifest v0.20.0 in docker.io/prom/alertmanager: toomanyrequests: You have reached your pull rate limit. You may increase the limit by authenticating and upgrading: https://www.docker.com/increase-rate-limit" 2023-11-17T05:27:23.778 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:26:22 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: stat: stderr time="2023-11-17T05:22:37Z" level=warning msg="Failed, retrying in 1s ... (2/3). Error: initializing source docker://prom/alertmanager:v0.20.0: (Mirrors also failed: [docker-mirror.front.sepia.ceph.com:5000/prom/alertmanager:v0.20.0: pinging container registry docker-mirror.front.sepia.ceph.com:5000: Get \"https://docker-mirror.front.sepia.ceph.com:5000/v2/\": dial tcp 172.21.0.79:5000: connect: no route to host]): docker.io/prom/alertmanager:v0.20.0: reading manifest v0.20.0 in docker.io/prom/alertmanager: toomanyrequests: You have reached your pull rate limit. You may increase the limit by authenticating and upgrading: https://www.docker.com/increase-rate-limit" 2023-11-17T05:27:23.778 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:26:22 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: stat: stderr time="2023-11-17T05:23:14Z" level=warning msg="Failed, retrying in 1s ... (3/3). Error: initializing source docker://prom/alertmanager:v0.20.0: (Mirrors also failed: [docker-mirror.front.sepia.ceph.com:5000/prom/alertmanager:v0.20.0: pinging container registry docker-mirror.front.sepia.ceph.com:5000: Get \"https://docker-mirror.front.sepia.ceph.com:5000/v2/\": dial tcp 172.21.0.79:5000: connect: no route to host]): docker.io/prom/alertmanager:v0.20.0: reading manifest v0.20.0 in docker.io/prom/alertmanager: toomanyrequests: You have reached your pull rate limit. You may increase the limit by authenticating and upgrading: https://www.docker.com/increase-rate-limit" 2023-11-17T05:27:23.778 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:26:22 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: stat: stderr Error: copying system image from manifest list: determining manifest MIME type for docker://prom/alertmanager:v0.20.0: reading manifest sha256:b9323917a2eda265bec69e59a457f001c529facbbc8166df277f4850cdac61a0 in docker.io/prom/alertmanager: toomanyrequests: You have reached your pull rate limit. You may increase the limit by authenticating and upgrading: https://www.docker.com/increase-rate-limit 2023-11-17T05:27:23.778 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:26:22 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: Non-zero exit code 125 from /bin/podman run --rm --ipc=host --no-hosts --net=host --entrypoint stat --init -e CONTAINER_IMAGE=docker.io/prom/alertmanager:v0.20.0 -e NODE_NAME=smithi040 -e CEPH_USE_RANDOM_NONCE=1 docker.io/prom/alertmanager:v0.20.0 -c %u %g /etc/prometheus 2023-11-17T05:27:23.778 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:26:22 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: stat: stderr Trying to pull docker.io/prom/alertmanager:v0.20.0... 2023-11-17T05:27:23.778 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:26:22 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: stat: stderr time="2023-11-17T05:24:28Z" level=warning msg="Failed, retrying in 1s ... (1/3). Error: initializing source docker://prom/alertmanager:v0.20.0: (Mirrors also failed: [docker-mirror.front.sepia.ceph.com:5000/prom/alertmanager:v0.20.0: pinging container registry docker-mirror.front.sepia.ceph.com:5000: Get \"https://docker-mirror.front.sepia.ceph.com:5000/v2/\": dial tcp 172.21.0.79:5000: connect: no route to host]): docker.io/prom/alertmanager:v0.20.0: reading manifest v0.20.0 in docker.io/prom/alertmanager: toomanyrequests: You have reached your pull rate limit. You may increase the limit by authenticating and upgrading: https://www.docker.com/increase-rate-limit" 2023-11-17T05:27:23.778 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:26:22 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: stat: stderr time="2023-11-17T05:25:05Z" level=warning msg="Failed, retrying in 1s ... (2/3). Error: initializing source docker://prom/alertmanager:v0.20.0: (Mirrors also failed: [docker-mirror.front.sepia.ceph.com:5000/prom/alertmanager:v0.20.0: pinging container registry docker-mirror.front.sepia.ceph.com:5000: Get \"https://docker-mirror.front.sepia.ceph.com:5000/v2/\": dial tcp 172.21.0.79:5000: connect: no route to host]): docker.io/prom/alertmanager:v0.20.0: reading manifest v0.20.0 in docker.io/prom/alertmanager: toomanyrequests: You have reached your pull rate limit. You may increase the limit by authenticating and upgrading: https://www.docker.com/increase-rate-limit" 2023-11-17T05:27:23.778 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:26:22 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: stat: stderr time="2023-11-17T05:25:43Z" level=warning msg="Failed, retrying in 1s ... (3/3). Error: initializing source docker://prom/alertmanager:v0.20.0: (Mirrors also failed: [docker-mirror.front.sepia.ceph.com:5000/prom/alertmanager:v0.20.0: pinging container registry docker-mirror.front.sepia.ceph.com:5000: Get \"https://docker-mirror.front.sepia.ceph.com:5000/v2/\": dial tcp 172.21.0.79:5000: connect: no route to host]): docker.io/prom/alertmanager:v0.20.0: reading manifest v0.20.0 in docker.io/prom/alertmanager: toomanyrequests: You have reached your pull rate limit. You may increase the limit by authenticating and upgrading: https://www.docker.com/increase-rate-limit" 2023-11-17T05:27:23.778 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:26:22 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: stat: stderr Error: initializing source docker://prom/alertmanager:v0.20.0: (Mirrors also failed: [docker-mirror.front.sepia.ceph.com:5000/prom/alertmanager:v0.20.0: pinging container registry docker-mirror.front.sepia.ceph.com:5000: Get "https://docker-mirror.front.sepia.ceph.com:5000/v2/": dial tcp 172.21.0.79:5000: connect: no route to host]): docker.io/prom/alertmanager:v0.20.0: reading manifest v0.20.0 in docker.io/prom/alertmanager: toomanyrequests: You have reached your pull rate limit. You may increase the limit by authenticating and upgrading: https://www.docker.com/increase-rate-limit 2023-11-17T05:27:23.778 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:26:22 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: Traceback (most recent call last): 2023-11-17T05:27:23.778 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:26:22 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: File "/var/lib/ceph/ed6470e2-8508-11ee-8dbe-212e2dc638e7/cephadm.30cb78bdbbafb384af862e1c2292b944f15942b586128e91262b43e91e11ae90", line 8029, in 2023-11-17T05:27:23.779 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:26:22 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: main() 2023-11-17T05:27:23.779 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:26:22 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: File "/var/lib/ceph/ed6470e2-8508-11ee-8dbe-212e2dc638e7/cephadm.30cb78bdbbafb384af862e1c2292b944f15942b586128e91262b43e91e11ae90", line 8017, in main 2023-11-17T05:27:23.779 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:26:22 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: r = ctx.func(ctx) 2023-11-17T05:27:23.779 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:26:22 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: File "/var/lib/ceph/ed6470e2-8508-11ee-8dbe-212e2dc638e7/cephadm.30cb78bdbbafb384af862e1c2292b944f15942b586128e91262b43e91e11ae90", line 1760, in _default_image 2023-11-17T05:27:23.779 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:26:22 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: return func(ctx) 2023-11-17T05:27:23.779 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:26:22 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: File "/var/lib/ceph/ed6470e2-8508-11ee-8dbe-212e2dc638e7/cephadm.30cb78bdbbafb384af862e1c2292b944f15942b586128e91262b43e91e11ae90", line 4263, in command_deploy 2023-11-17T05:27:23.779 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:26:22 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: uid, gid = extract_uid_gid_monitoring(ctx, daemon_type) 2023-11-17T05:27:23.779 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:26:22 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: File "/var/lib/ceph/ed6470e2-8508-11ee-8dbe-212e2dc638e7/cephadm.30cb78bdbbafb384af862e1c2292b944f15942b586128e91262b43e91e11ae90", line 4191, in extract_uid_gid_monitoring 2023-11-17T05:27:23.779 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:26:22 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: uid, gid = extract_uid_gid(ctx, file_path=['/etc/alertmanager', '/etc/prometheus']) 2023-11-17T05:27:23.779 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:26:22 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: File "/var/lib/ceph/ed6470e2-8508-11ee-8dbe-212e2dc638e7/cephadm.30cb78bdbbafb384af862e1c2292b944f15942b586128e91262b43e91e11ae90", line 2531, in extract_uid_gid 2023-11-17T05:27:23.779 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:26:22 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: raise RuntimeError('uid/gid not found') 2023-11-17T05:27:23.779 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:26:22 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: RuntimeError: uid/gid not found 2023-11-17T05:27:23.779 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:26:22 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: Traceback (most recent call last): 2023-11-17T05:27:23.779 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:26:22 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1144, in _remote_connection 2023-11-17T05:27:23.780 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:26:22 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: yield (conn, connr) 2023-11-17T05:27:23.780 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:26:22 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1059, in _run_cephadm 2023-11-17T05:27:23.780 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:26:22 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: code, '\n'.join(err))) 2023-11-17T05:27:23.780 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:26:22 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr:Deploy daemon alertmanager.smithi040 ... 2023-11-17T05:27:23.780 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:26:22 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: Non-zero exit code 125 from /bin/podman run --rm --ipc=host --no-hosts --net=host --entrypoint stat --init -e CONTAINER_IMAGE=docker.io/prom/alertmanager:v0.20.0 -e NODE_NAME=smithi040 -e CEPH_USE_RANDOM_NONCE=1 docker.io/prom/alertmanager:v0.20.0 -c %u %g /etc/alertmanager 2023-11-17T05:27:23.780 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:26:22 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: stat: stderr Trying to pull docker.io/prom/alertmanager:v0.20.0... 2023-11-17T05:27:23.780 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:26:22 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: stat: stderr time="2023-11-17T05:21:59Z" level=warning msg="Failed, retrying in 1s ... (1/3). Error: initializing source docker://prom/alertmanager:v0.20.0: (Mirrors also failed: [docker-mirror.front.sepia.ceph.com:5000/prom/alertmanager:v0.20.0: pinging container registry docker-mirror.front.sepia.ceph.com:5000: Get \"https://docker-mirror.front.sepia.ceph.com:5000/v2/\": dial tcp 172.21.0.79:5000: connect: no route to host]): docker.io/prom/alertmanager:v0.20.0: reading manifest v0.20.0 in docker.io/prom/alertmanager: toomanyrequests: You have reached your pull rate limit. You may increase the limit by authenticating and upgrading: https://www.docker.com/increase-rate-limit" 2023-11-17T05:27:23.780 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:26:22 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: stat: stderr time="2023-11-17T05:22:37Z" level=warning msg="Failed, retrying in 1s ... (2/3). Error: initializing source docker://prom/alertmanager:v0.20.0: (Mirrors also failed: [docker-mirror.front.sepia.ceph.com:5000/prom/alertmanager:v0.20.0: pinging container registry docker-mirror.front.sepia.ceph.com:5000: Get \"https://docker-mirror.front.sepia.ceph.com:5000/v2/\": dial tcp 172.21.0.79:5000: connect: no route to host]): docker.io/prom/alertmanager:v0.20.0: reading manifest v0.20.0 in docker.io/prom/alertmanager: toomanyrequests: You have reached your pull rate limit. You may increase the limit by authenticating and upgrading: https://www.docker.com/increase-rate-limit" 2023-11-17T05:27:23.812 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:26:22 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: stat: stderr time="2023-11-17T05:23:14Z" level=warning msg="Failed, retrying in 1s ... (3/3). Error: initializing source docker://prom/alertmanager:v0.20.0: (Mirrors also failed: [docker-mirror.front.sepia.ceph.com:5000/prom/alertmanager:v0.20.0: pinging container registry docker-mirror.front.sepia.ceph.com:5000: Get \"https://docker-mirror.front.sepia.ceph.com:5000/v2/\": dial tcp 172.21.0.79:5000: connect: no route to host]): docker.io/prom/alertmanager:v0.20.0: reading manifest v0.20.0 in docker.io/prom/alertmanager: toomanyrequests: You have reached your pull rate limit. You may increase the limit by authenticating and upgrading: https://www.docker.com/increase-rate-limit" 2023-11-17T05:27:23.812 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:26:22 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: stat: stderr Error: copying system image from manifest list: determining manifest MIME type for docker://prom/alertmanager:v0.20.0: reading manifest sha256:b9323917a2eda265bec69e59a457f001c529facbbc8166df277f4850cdac61a0 in docker.io/prom/alertmanager: toomanyrequests: You have reached your pull rate limit. You may increase the limit by authenticating and upgrading: https://www.docker.com/increase-rate-limit 2023-11-17T05:27:23.812 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:26:22 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: Non-zero exit code 125 from /bin/podman run --rm --ipc=host --no-hosts --net=host --entrypoint stat --init -e CONTAINER_IMAGE=docker.io/prom/alertmanager:v0.20.0 -e NODE_NAME=smithi040 -e CEPH_USE_RANDOM_NONCE=1 docker.io/prom/alertmanager:v0.20.0 -c %u %g /etc/prometheus 2023-11-17T05:27:23.812 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:26:22 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: stat: stderr Trying to pull docker.io/prom/alertmanager:v0.20.0... 2023-11-17T05:27:23.812 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:26:22 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: stat: stderr time="2023-11-17T05:24:28Z" level=warning msg="Failed, retrying in 1s ... (1/3). Error: initializing source docker://prom/alertmanager:v0.20.0: (Mirrors also failed: [docker-mirror.front.sepia.ceph.com:5000/prom/alertmanager:v0.20.0: pinging container registry docker-mirror.front.sepia.ceph.com:5000: Get \"https://docker-mirror.front.sepia.ceph.com:5000/v2/\": dial tcp 172.21.0.79:5000: connect: no route to host]): docker.io/prom/alertmanager:v0.20.0: reading manifest v0.20.0 in docker.io/prom/alertmanager: toomanyrequests: You have reached your pull rate limit. You may increase the limit by authenticating and upgrading: https://www.docker.com/increase-rate-limit" 2023-11-17T05:27:23.812 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:26:22 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: stat: stderr time="2023-11-17T05:25:05Z" level=warning msg="Failed, retrying in 1s ... (2/3). Error: initializing source docker://prom/alertmanager:v0.20.0: (Mirrors also failed: [docker-mirror.front.sepia.ceph.com:5000/prom/alertmanager:v0.20.0: pinging container registry docker-mirror.front.sepia.ceph.com:5000: Get \"https://docker-mirror.front.sepia.ceph.com:5000/v2/\": dial tcp 172.21.0.79:5000: connect: no route to host]): docker.io/prom/alertmanager:v0.20.0: reading manifest v0.20.0 in docker.io/prom/alertmanager: toomanyrequests: You have reached your pull rate limit. You may increase the limit by authenticating and upgrading: https://www.docker.com/increase-rate-limit" 2023-11-17T05:27:23.812 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:26:22 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: stat: stderr time="2023-11-17T05:25:43Z" level=warning msg="Failed, retrying in 1s ... (3/3). Error: initializing source docker://prom/alertmanager:v0.20.0: (Mirrors also failed: [docker-mirror.front.sepia.ceph.com:5000/prom/alertmanager:v0.20.0: pinging container registry docker-mirror.front.sepia.ceph.com:5000: Get \"https://docker-mirror.front.sepia.ceph.com:5000/v2/\": dial tcp 172.21.0.79:5000: connect: no route to host]): docker.io/prom/alertmanager:v0.20.0: reading manifest v0.20.0 in docker.io/prom/alertmanager: toomanyrequests: You have reached your pull rate limit. You may increase the limit by authenticating and upgrading: https://www.docker.com/increase-rate-limit" 2023-11-17T05:27:23.812 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:26:22 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: stat: stderr Error: initializing source docker://prom/alertmanager:v0.20.0: (Mirrors also failed: [docker-mirror.front.sepia.ceph.com:5000/prom/alertmanager:v0.20.0: pinging container registry docker-mirror.front.sepia.ceph.com:5000: Get "https://docker-mirror.front.sepia.ceph.com:5000/v2/": dial tcp 172.21.0.79:5000: connect: no route to host]): docker.io/prom/alertmanager:v0.20.0: reading manifest v0.20.0 in docker.io/prom/alertmanager: toomanyrequests: You have reached your pull rate limit. You may increase the limit by authenticating and upgrading: https://www.docker.com/increase-rate-limit 2023-11-17T05:27:23.812 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:26:22 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: Traceback (most recent call last): 2023-11-17T05:27:23.813 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:26:22 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: File "/var/lib/ceph/ed6470e2-8508-11ee-8dbe-212e2dc638e7/cephadm.30cb78bdbbafb384af862e1c2292b944f15942b586128e91262b43e91e11ae90", line 8029, in 2023-11-17T05:27:23.813 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:26:22 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: main() 2023-11-17T05:27:23.813 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:26:22 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: File "/var/lib/ceph/ed6470e2-8508-11ee-8dbe-212e2dc638e7/cephadm.30cb78bdbbafb384af862e1c2292b944f15942b586128e91262b43e91e11ae90", line 8017, in main 2023-11-17T05:27:23.813 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:26:22 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: r = ctx.func(ctx) 2023-11-17T05:27:23.813 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:26:22 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: File "/var/lib/ceph/ed6470e2-8508-11ee-8dbe-212e2dc638e7/cephadm.30cb78bdbbafb384af862e1c2292b944f15942b586128e91262b43e91e11ae90", line 1760, in _default_image 2023-11-17T05:27:23.813 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:26:22 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: return func(ctx) 2023-11-17T05:27:23.813 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:26:22 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: File "/var/lib/ceph/ed6470e2-8508-11ee-8dbe-212e2dc638e7/cephadm.30cb78bdbbafb384af862e1c2292b944f15942b586128e91262b43e91e11ae90", line 4263, in command_deploy 2023-11-17T05:27:23.813 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:26:22 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: uid, gid = extract_uid_gid_monitoring(ctx, daemon_type) 2023-11-17T05:27:23.813 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:26:22 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: File "/var/lib/ceph/ed6470e2-8508-11ee-8dbe-212e2dc638e7/cephadm.30cb78bdbbafb384af862e1c2292b944f15942b586128e91262b43e91e11ae90", line 4191, in extract_uid_gid_monitoring 2023-11-17T05:27:23.813 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:26:22 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: uid, gid = extract_uid_gid(ctx, file_path=['/etc/alertmanager', '/etc/prometheus']) 2023-11-17T05:27:23.813 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:26:22 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: File "/var/lib/ceph/ed6470e2-8508-11ee-8dbe-212e2dc638e7/cephadm.30cb78bdbbafb384af862e1c2292b944f15942b586128e91262b43e91e11ae90", line 2531, in extract_uid_gid 2023-11-17T05:27:23.813 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:26:22 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: raise RuntimeError('uid/gid not found') 2023-11-17T05:27:23.813 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:26:22 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: RuntimeError: uid/gid not found 2023-11-17T05:27:23.814 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:26:22 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cephadm 2023-11-17T05:26:20.739549+0000 mgr.smithi040.ogcprl (mgr.14164) 157 : cephadm [INF] Deploying daemon crash.smithi040 on smithi040 2023-11-17T05:27:23.814 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:26:22 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:26:21.204874+0000 mgr.smithi040.ogcprl (mgr.14164) 158 : cluster [DBG] pgmap v149: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:27:23.814 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:26:22 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:26:22.286672+0000 mon.smithi040 (mon.0) 194 : audit [INF] from='mgr.14164 172.21.15.40:0/2203032451' entity='mgr.smithi040.ogcprl' 2023-11-17T05:27:23.814 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:26:23 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:26:22.809704+0000 mon.smithi040 (mon.0) 195 : audit [INF] from='mgr.14164 172.21.15.40:0/2203032451' entity='mgr.smithi040.ogcprl' 2023-11-17T05:27:23.814 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:26:23 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:26:22.811702+0000 mon.smithi040 (mon.0) 196 : audit [INF] from='mgr.14164 172.21.15.40:0/2203032451' entity='mgr.smithi040.ogcprl' 2023-11-17T05:27:23.814 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:26:23 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:26:22.812156+0000 mon.smithi040 (mon.0) 197 : audit [INF] from='mgr.14164 172.21.15.40:0/2203032451' entity='mgr.smithi040.ogcprl' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2023-11-17T05:27:23.814 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:26:23 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:26:22.812637+0000 mgr.smithi040.ogcprl (mgr.14164) 159 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2023-11-17T05:27:23.814 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:26:23 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:26:22.815934+0000 mon.smithi040 (mon.0) 198 : audit [INF] from='mgr.14164 172.21.15.40:0/2203032451' entity='mgr.smithi040.ogcprl' 2023-11-17T05:27:23.814 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:26:23 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cephadm 2023-11-17T05:26:22.819389+0000 mgr.smithi040.ogcprl (mgr.14164) 160 : cephadm [INF] Deploying daemon grafana.smithi040 on smithi040 2023-11-17T05:27:23.814 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:26:23 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:26:23.205278+0000 mgr.smithi040.ogcprl (mgr.14164) 161 : cluster [DBG] pgmap v150: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:27:23.814 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:26:26 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:26:25.205613+0000 mgr.smithi040.ogcprl (mgr.14164) 162 : cluster [DBG] pgmap v151: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:27:23.814 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:26:28 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:26:27.205827+0000 mgr.smithi040.ogcprl (mgr.14164) 163 : cluster [DBG] pgmap v152: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:27:23.814 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:26:30 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:26:29.206150+0000 mgr.smithi040.ogcprl (mgr.14164) 164 : cluster [DBG] pgmap v153: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:27:23.815 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:26:32 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:26:31.206414+0000 mgr.smithi040.ogcprl (mgr.14164) 165 : cluster [DBG] pgmap v154: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:27:23.815 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:26:34 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:26:33.206643+0000 mgr.smithi040.ogcprl (mgr.14164) 166 : cluster [DBG] pgmap v155: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:27:23.815 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:26:36 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:26:35.206885+0000 mgr.smithi040.ogcprl (mgr.14164) 167 : cluster [DBG] pgmap v156: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:27:23.815 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:26:38 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:26:37.207208+0000 mgr.smithi040.ogcprl (mgr.14164) 168 : cluster [DBG] pgmap v157: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:27:23.815 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:26:40 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:26:39.207446+0000 mgr.smithi040.ogcprl (mgr.14164) 169 : cluster [DBG] pgmap v158: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:27:23.815 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:26:40 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:26:39.744311+0000 mon.smithi040 (mon.0) 199 : audit [INF] from='mgr.14164 172.21.15.40:0/2203032451' entity='mgr.smithi040.ogcprl' 2023-11-17T05:27:23.883 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:26:41 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cephadm 2023-11-17T05:26:39.746339+0000 mgr.smithi040.ogcprl (mgr.14164) 170 : cephadm [INF] Deploying daemon node-exporter.smithi040 on smithi040 2023-11-17T05:27:23.883 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:26:42 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:26:41.208034+0000 mgr.smithi040.ogcprl (mgr.14164) 171 : cluster [DBG] pgmap v159: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:27:23.883 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:26:44 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:26:43.208336+0000 mgr.smithi040.ogcprl (mgr.14164) 172 : cluster [DBG] pgmap v160: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:27:23.883 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:26:46 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:26:45.208581+0000 mgr.smithi040.ogcprl (mgr.14164) 173 : cluster [DBG] pgmap v161: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:27:23.883 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:26:48 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:26:47.208837+0000 mgr.smithi040.ogcprl (mgr.14164) 174 : cluster [DBG] pgmap v162: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:27:23.883 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:26:50 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:26:49.209241+0000 mgr.smithi040.ogcprl (mgr.14164) 175 : cluster [DBG] pgmap v163: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:27:23.883 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:26:52 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:26:51.209720+0000 mgr.smithi040.ogcprl (mgr.14164) 176 : cluster [DBG] pgmap v164: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:27:23.883 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:26:54 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:26:53.210168+0000 mgr.smithi040.ogcprl (mgr.14164) 177 : cluster [DBG] pgmap v165: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:27:23.883 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:26:56 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:26:55.210556+0000 mgr.smithi040.ogcprl (mgr.14164) 178 : cluster [DBG] pgmap v166: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:27:23.884 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:26:58 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:26:57.210863+0000 mgr.smithi040.ogcprl (mgr.14164) 179 : cluster [DBG] pgmap v167: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:27:23.884 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:27:00 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:26:59.211259+0000 mgr.smithi040.ogcprl (mgr.14164) 180 : cluster [DBG] pgmap v168: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:27:23.884 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:27:02 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:27:01.211643+0000 mgr.smithi040.ogcprl (mgr.14164) 181 : cluster [DBG] pgmap v169: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:27:23.884 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:27:04 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:27:03.211936+0000 mgr.smithi040.ogcprl (mgr.14164) 182 : cluster [DBG] pgmap v170: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:27:23.884 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:27:06 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:27:05.212227+0000 mgr.smithi040.ogcprl (mgr.14164) 183 : cluster [DBG] pgmap v171: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:27:23.884 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:27:08 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:27:07.212709+0000 mgr.smithi040.ogcprl (mgr.14164) 184 : cluster [DBG] pgmap v172: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:27:23.884 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:27:10 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:27:09.213037+0000 mgr.smithi040.ogcprl (mgr.14164) 185 : cluster [DBG] pgmap v173: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:27:23.884 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:27:11 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:27:11.313429+0000 mon.smithi040 (mon.0) 200 : audit [INF] from='mgr.14164 172.21.15.40:0/2203032451' entity='mgr.smithi040.ogcprl' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi040.ogcprl/mirror_snapshot_schedule"}]: dispatch 2023-11-17T05:27:23.884 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:27:11 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:27:11.314829+0000 mon.smithi040 (mon.0) 201 : audit [INF] from='mgr.14164 172.21.15.40:0/2203032451' entity='mgr.smithi040.ogcprl' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi040.ogcprl/trash_purge_schedule"}]: dispatch 2023-11-17T05:27:23.884 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:27:12 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:27:11.213308+0000 mgr.smithi040.ogcprl (mgr.14164) 186 : cluster [DBG] pgmap v174: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:27:23.884 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:27:14 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:27:13.213784+0000 mgr.smithi040.ogcprl (mgr.14164) 187 : cluster [DBG] pgmap v175: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:27:23.884 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:27:16 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:27:15.214144+0000 mgr.smithi040.ogcprl (mgr.14164) 188 : cluster [DBG] pgmap v176: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:27:23.884 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:27:18 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:27:17.214545+0000 mgr.smithi040.ogcprl (mgr.14164) 189 : cluster [DBG] pgmap v177: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:27:23.885 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:27:20 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:27:19.214995+0000 mgr.smithi040.ogcprl (mgr.14164) 190 : cluster [DBG] pgmap v178: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:27:23.885 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:27:22 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:27:21.215350+0000 mgr.smithi040.ogcprl (mgr.14164) 191 : cluster [DBG] pgmap v179: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:27:24.713 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2023-11-17T05:27:24.713 DEBUG:teuthology.orchestra.run.smithi066:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.4 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ed6470e2-8508-11ee-8dbe-212e2dc638e7 -- ceph mon dump -f json 2023-11-17T05:27:24.988 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:27:24 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:27:23.215824+0000 mgr.smithi040.ogcprl (mgr.14164) 192 : cluster [DBG] pgmap v180: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:27:25.892 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:25.890+0000 7f808332b700 1 -- 172.21.15.66:0/3333240748 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f807c0f62a0 msgr2=0x7f807c0f66c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:27:25.892 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:25.890+0000 7f808332b700 1 --2- 172.21.15.66:0/3333240748 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f807c0f62a0 0x7f807c0f66c0 secure :-1 s=READY pgs=215 cs=0 l=1 rev1=1 rx=0x7f8078009fb0 tx=0x7f8078009b30).stop 2023-11-17T05:27:25.892 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:25.890+0000 7f808332b700 1 -- 172.21.15.66:0/3333240748 shutdown_connections 2023-11-17T05:27:25.892 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:25.890+0000 7f808332b700 1 --2- 172.21.15.66:0/3333240748 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f807c0f62a0 0x7f807c0f66c0 unknown :-1 s=CLOSED pgs=215 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:27:25.892 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:25.890+0000 7f808332b700 1 -- 172.21.15.66:0/3333240748 >> 172.21.15.66:0/3333240748 conn(0x7f807c0f1aa0 msgr2=0x7f807c0f3ec0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:27:25.892 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:25.890+0000 7f808332b700 1 -- 172.21.15.66:0/3333240748 shutdown_connections 2023-11-17T05:27:25.892 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:25.890+0000 7f808332b700 1 -- 172.21.15.66:0/3333240748 wait complete. 2023-11-17T05:27:25.892 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:25.890+0000 7f808332b700 1 Processor -- start 2023-11-17T05:27:25.893 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:25.891+0000 7f808332b700 1 -- start start 2023-11-17T05:27:25.893 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:25.891+0000 7f808332b700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f807c0f62a0 0x7f807c1064b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:27:25.893 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:25.891+0000 7f808332b700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f807c1069a0 con 0x7f807c0f62a0 2023-11-17T05:27:25.894 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:25.891+0000 7f8082329700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f807c0f62a0 0x7f807c1064b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:27:25.894 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:25.891+0000 7f8082329700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f807c0f62a0 0x7f807c1064b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.66:47914/0 (socket says 172.21.15.66:47914) 2023-11-17T05:27:25.894 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:25.891+0000 7f8082329700 1 -- 172.21.15.66:0/2842693404 learned_addr learned my addr 172.21.15.66:0/2842693404 (peer_addr_for_me v2:172.21.15.66:0/0) 2023-11-17T05:27:25.894 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:25.891+0000 7f8082329700 1 -- 172.21.15.66:0/2842693404 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f8078005040 con 0x7f807c0f62a0 2023-11-17T05:27:25.894 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:25.891+0000 7f8082329700 1 --2- 172.21.15.66:0/2842693404 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f807c0f62a0 0x7f807c1064b0 secure :-1 s=READY pgs=216 cs=0 l=1 rev1=1 rx=0x7f8078013510 tx=0x7f807800b040).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:27:25.894 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:25.892+0000 7f8072ffd700 1 -- 172.21.15.66:0/2842693404 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f8078009320 con 0x7f807c0f62a0 2023-11-17T05:27:25.894 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:25.892+0000 7f808332b700 1 -- 172.21.15.66:0/2842693404 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f807c106ba0 con 0x7f807c0f62a0 2023-11-17T05:27:25.894 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:25.892+0000 7f808332b700 1 -- 172.21.15.66:0/2842693404 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f807c107030 con 0x7f807c0f62a0 2023-11-17T05:27:25.894 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:25.892+0000 7f8072ffd700 1 -- 172.21.15.66:0/2842693404 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(25 keys) v1 ==== 1074+0+0 (secure 0 0 0) 0x7f8078013f00 con 0x7f807c0f62a0 2023-11-17T05:27:25.895 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:25.892+0000 7f8072ffd700 1 -- 172.21.15.66:0/2842693404 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f8078014dc0 con 0x7f807c0f62a0 2023-11-17T05:27:25.895 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:25.892+0000 7f8072ffd700 1 -- 172.21.15.66:0/2842693404 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 17) v1 ==== 36269+0+0 (secure 0 0 0) 0x7f8078014420 con 0x7f807c0f62a0 2023-11-17T05:27:25.895 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:25.892+0000 7f808332b700 1 -- 172.21.15.66:0/2842693404 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f807c045ad0 con 0x7f807c0f62a0 2023-11-17T05:27:25.895 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:25.893+0000 7f8072ffd700 1 --2- 172.21.15.66:0/2842693404 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f806802ffb0 0x7f8068032470 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:27:25.895 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:25.893+0000 7f8072ffd700 1 -- 172.21.15.66:0/2842693404 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(5..5 src has 1..5) v4 ==== 1270+0+0 (secure 0 0 0) 0x7f8078041130 con 0x7f807c0f62a0 2023-11-17T05:27:25.895 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:25.893+0000 7f8081b28700 1 --2- 172.21.15.66:0/2842693404 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f806802ffb0 0x7f8068032470 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:27:25.895 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:25.893+0000 7f8081b28700 1 --2- 172.21.15.66:0/2842693404 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f806802ffb0 0x7f8068032470 secure :-1 s=READY pgs=73 cs=0 l=1 rev1=1 rx=0x7f806c0096a0 tx=0x7f806c006b40).ready entity=mgr.14164 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:27:25.899 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:25.898+0000 7f8072ffd700 1 -- 172.21.15.66:0/2842693404 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+152212 (secure 0 0 0) 0x7f80780095f0 con 0x7f807c0f62a0 2023-11-17T05:27:26.117 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:26.116+0000 7f808332b700 1 -- 172.21.15.66:0/2842693404 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "mon dump", "format": "json"} v 0) v1 -- 0x7f807c0fbce0 con 0x7f807c0f62a0 2023-11-17T05:27:26.118 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:26.116+0000 7f8072ffd700 1 -- 172.21.15.66:0/2842693404 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mon_command_ack([{"prefix": "mon dump", "format": "json"}]=0 dumped monmap epoch 1 v1) v1 ==== 95+0+690 (secure 0 0 0) 0x7f807801db70 con 0x7f807c0f62a0 2023-11-17T05:27:26.118 INFO:teuthology.orchestra.run.smithi066.stdout: 2023-11-17T05:27:26.118 INFO:teuthology.orchestra.run.smithi066.stdout:{"epoch":1,"fsid":"ed6470e2-8508-11ee-8dbe-212e2dc638e7","modified":"2023-11-17T05:20:13.934038Z","created":"2023-11-17T05:20:13.934038Z","min_mon_release":16,"min_mon_release_name":"pacific","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging"],"optional":[]},"mons":[{"rank":0,"name":"smithi040","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:3300","nonce":0},{"type":"v1","addr":"172.21.15.40:6789","nonce":0}]},"addr":"172.21.15.40:6789/0","public_addr":"172.21.15.40:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2023-11-17T05:27:26.119 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:26.118+0000 7f808332b700 1 -- 172.21.15.66:0/2842693404 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f806802ffb0 msgr2=0x7f8068032470 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:27:26.120 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:26.118+0000 7f808332b700 1 --2- 172.21.15.66:0/2842693404 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f806802ffb0 0x7f8068032470 secure :-1 s=READY pgs=73 cs=0 l=1 rev1=1 rx=0x7f806c0096a0 tx=0x7f806c006b40).stop 2023-11-17T05:27:26.120 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:26.118+0000 7f808332b700 1 -- 172.21.15.66:0/2842693404 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f807c0f62a0 msgr2=0x7f807c1064b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:27:26.120 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:26.118+0000 7f808332b700 1 --2- 172.21.15.66:0/2842693404 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f807c0f62a0 0x7f807c1064b0 secure :-1 s=READY pgs=216 cs=0 l=1 rev1=1 rx=0x7f8078013510 tx=0x7f807800b040).stop 2023-11-17T05:27:26.120 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:26.118+0000 7f808332b700 1 -- 172.21.15.66:0/2842693404 shutdown_connections 2023-11-17T05:27:26.120 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:26.118+0000 7f808332b700 1 --2- 172.21.15.66:0/2842693404 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f806802ffb0 0x7f8068032470 unknown :-1 s=CLOSED pgs=73 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:27:26.120 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:26.118+0000 7f808332b700 1 --2- 172.21.15.66:0/2842693404 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f807c0f62a0 0x7f807c1064b0 unknown :-1 s=CLOSED pgs=216 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:27:26.120 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:26.118+0000 7f808332b700 1 -- 172.21.15.66:0/2842693404 >> 172.21.15.66:0/2842693404 conn(0x7f807c0f1aa0 msgr2=0x7f807c0f2510 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:27:26.120 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:26.118+0000 7f808332b700 1 -- 172.21.15.66:0/2842693404 shutdown_connections 2023-11-17T05:27:26.120 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:26.118+0000 7f808332b700 1 -- 172.21.15.66:0/2842693404 wait complete. 2023-11-17T05:27:26.120 INFO:teuthology.orchestra.run.smithi066.stderr:dumped monmap epoch 1 2023-11-17T05:27:26.989 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:27:26 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:27:25.216197+0000 mgr.smithi040.ogcprl (mgr.14164) 193 : cluster [DBG] pgmap v181: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:27:26.989 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:27:26 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:27:26.118227+0000 mon.smithi040 (mon.0) 202 : audit [DBG] from='client.? 172.21.15.66:0/2842693404' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-11-17T05:27:27.507 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2023-11-17T05:27:27.507 DEBUG:teuthology.orchestra.run.smithi066:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.4 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ed6470e2-8508-11ee-8dbe-212e2dc638e7 -- ceph mon dump -f json 2023-11-17T05:27:28.852 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:28.851+0000 7fb88551e700 1 -- 172.21.15.66:0/1899237098 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fb8800f51a0 msgr2=0x7fb8800f55c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:27:28.852 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:28.851+0000 7fb88551e700 1 --2- 172.21.15.66:0/1899237098 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fb8800f51a0 0x7fb8800f55c0 secure :-1 s=READY pgs=217 cs=0 l=1 rev1=1 rx=0x7fb870009fb0 tx=0x7fb870009b30).stop 2023-11-17T05:27:28.852 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:28.851+0000 7fb88551e700 1 -- 172.21.15.66:0/1899237098 shutdown_connections 2023-11-17T05:27:28.852 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:28.851+0000 7fb88551e700 1 --2- 172.21.15.66:0/1899237098 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fb8800f51a0 0x7fb8800f55c0 unknown :-1 s=CLOSED pgs=217 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:27:28.852 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:28.851+0000 7fb88551e700 1 -- 172.21.15.66:0/1899237098 >> 172.21.15.66:0/1899237098 conn(0x7fb8800f09a0 msgr2=0x7fb8800f2dc0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:27:28.853 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:28.851+0000 7fb88551e700 1 -- 172.21.15.66:0/1899237098 shutdown_connections 2023-11-17T05:27:28.853 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:28.851+0000 7fb88551e700 1 -- 172.21.15.66:0/1899237098 wait complete. 2023-11-17T05:27:28.854 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:28.851+0000 7fb88551e700 1 Processor -- start 2023-11-17T05:27:28.854 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:28.851+0000 7fb88551e700 1 -- start start 2023-11-17T05:27:28.854 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:28.852+0000 7fb88551e700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fb8800f51a0 0x7fb8801053b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:27:28.854 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:28.852+0000 7fb88551e700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fb8801058a0 con 0x7fb8800f51a0 2023-11-17T05:27:28.854 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:28.852+0000 7fb87ffff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fb8800f51a0 0x7fb8801053b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:27:28.854 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:28.852+0000 7fb87ffff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fb8800f51a0 0x7fb8801053b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.66:47918/0 (socket says 172.21.15.66:47918) 2023-11-17T05:27:28.854 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:28.852+0000 7fb87ffff700 1 -- 172.21.15.66:0/1820720930 learned_addr learned my addr 172.21.15.66:0/1820720930 (peer_addr_for_me v2:172.21.15.66:0/0) 2023-11-17T05:27:28.854 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:28.852+0000 7fb87ffff700 1 -- 172.21.15.66:0/1820720930 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fb870005040 con 0x7fb8800f51a0 2023-11-17T05:27:28.855 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:28.852+0000 7fb87ffff700 1 --2- 172.21.15.66:0/1820720930 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fb8800f51a0 0x7fb8801053b0 secure :-1 s=READY pgs=218 cs=0 l=1 rev1=1 rx=0x7fb870013510 tx=0x7fb87000b040).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:27:28.855 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:28.852+0000 7fb87cff9700 1 -- 172.21.15.66:0/1820720930 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7fb870009320 con 0x7fb8800f51a0 2023-11-17T05:27:28.855 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:28.852+0000 7fb87cff9700 1 -- 172.21.15.66:0/1820720930 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(25 keys) v1 ==== 1074+0+0 (secure 0 0 0) 0x7fb870013f00 con 0x7fb8800f51a0 2023-11-17T05:27:28.855 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:28.852+0000 7fb88551e700 1 -- 172.21.15.66:0/1820720930 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fb880105aa0 con 0x7fb8800f51a0 2023-11-17T05:27:28.855 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:28.853+0000 7fb87cff9700 1 -- 172.21.15.66:0/1820720930 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7fb870014dc0 con 0x7fb8800f51a0 2023-11-17T05:27:28.855 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:28.853+0000 7fb88551e700 1 -- 172.21.15.66:0/1820720930 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fb880105f30 con 0x7fb8800f51a0 2023-11-17T05:27:28.856 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:28.853+0000 7fb87cff9700 1 -- 172.21.15.66:0/1820720930 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 17) v1 ==== 36269+0+0 (secure 0 0 0) 0x7fb8700095f0 con 0x7fb8800f51a0 2023-11-17T05:27:28.856 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:28.853+0000 7fb88551e700 1 -- 172.21.15.66:0/1820720930 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fb880045ad0 con 0x7fb8800f51a0 2023-11-17T05:27:28.856 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:28.854+0000 7fb87cff9700 1 --2- 172.21.15.66:0/1820720930 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fb86802ff60 0x7fb868032420 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:27:28.856 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:28.854+0000 7fb87cff9700 1 -- 172.21.15.66:0/1820720930 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(5..5 src has 1..5) v4 ==== 1270+0+0 (secure 0 0 0) 0x7fb870041100 con 0x7fb8800f51a0 2023-11-17T05:27:28.856 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:28.854+0000 7fb87f7fe700 1 --2- 172.21.15.66:0/1820720930 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fb86802ff60 0x7fb868032420 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:27:28.856 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:28.854+0000 7fb87f7fe700 1 --2- 172.21.15.66:0/1820720930 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fb86802ff60 0x7fb868032420 secure :-1 s=READY pgs=74 cs=0 l=1 rev1=1 rx=0x7fb8740096a0 tx=0x7fb874006b40).ready entity=mgr.14164 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:27:28.860 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:28.858+0000 7fb87cff9700 1 -- 172.21.15.66:0/1820720930 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+152212 (secure 0 0 0) 0x7fb870014420 con 0x7fb8800f51a0 2023-11-17T05:27:28.988 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:27:28 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:27:27.216605+0000 mgr.smithi040.ogcprl (mgr.14164) 194 : cluster [DBG] pgmap v182: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:27:29.083 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:29.082+0000 7fb88551e700 1 -- 172.21.15.66:0/1820720930 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "mon dump", "format": "json"} v 0) v1 -- 0x7fb880106de0 con 0x7fb8800f51a0 2023-11-17T05:27:29.083 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:29.082+0000 7fb87cff9700 1 -- 172.21.15.66:0/1820720930 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mon_command_ack([{"prefix": "mon dump", "format": "json"}]=0 dumped monmap epoch 1 v1) v1 ==== 95+0+690 (secure 0 0 0) 0x7fb870010230 con 0x7fb8800f51a0 2023-11-17T05:27:29.084 INFO:teuthology.orchestra.run.smithi066.stdout: 2023-11-17T05:27:29.084 INFO:teuthology.orchestra.run.smithi066.stdout:{"epoch":1,"fsid":"ed6470e2-8508-11ee-8dbe-212e2dc638e7","modified":"2023-11-17T05:20:13.934038Z","created":"2023-11-17T05:20:13.934038Z","min_mon_release":16,"min_mon_release_name":"pacific","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging"],"optional":[]},"mons":[{"rank":0,"name":"smithi040","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:3300","nonce":0},{"type":"v1","addr":"172.21.15.40:6789","nonce":0}]},"addr":"172.21.15.40:6789/0","public_addr":"172.21.15.40:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2023-11-17T05:27:29.085 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:29.084+0000 7fb88551e700 1 -- 172.21.15.66:0/1820720930 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fb86802ff60 msgr2=0x7fb868032420 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:27:29.085 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:29.084+0000 7fb88551e700 1 --2- 172.21.15.66:0/1820720930 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fb86802ff60 0x7fb868032420 secure :-1 s=READY pgs=74 cs=0 l=1 rev1=1 rx=0x7fb8740096a0 tx=0x7fb874006b40).stop 2023-11-17T05:27:29.085 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:29.084+0000 7fb88551e700 1 -- 172.21.15.66:0/1820720930 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fb8800f51a0 msgr2=0x7fb8801053b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:27:29.085 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:29.084+0000 7fb88551e700 1 --2- 172.21.15.66:0/1820720930 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fb8800f51a0 0x7fb8801053b0 secure :-1 s=READY pgs=218 cs=0 l=1 rev1=1 rx=0x7fb870013510 tx=0x7fb87000b040).stop 2023-11-17T05:27:29.086 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:29.084+0000 7fb88551e700 1 -- 172.21.15.66:0/1820720930 shutdown_connections 2023-11-17T05:27:29.086 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:29.084+0000 7fb88551e700 1 --2- 172.21.15.66:0/1820720930 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fb86802ff60 0x7fb868032420 unknown :-1 s=CLOSED pgs=74 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:27:29.086 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:29.084+0000 7fb88551e700 1 --2- 172.21.15.66:0/1820720930 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fb8800f51a0 0x7fb8801053b0 unknown :-1 s=CLOSED pgs=218 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:27:29.086 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:29.084+0000 7fb88551e700 1 -- 172.21.15.66:0/1820720930 >> 172.21.15.66:0/1820720930 conn(0x7fb8800f09a0 msgr2=0x7fb8800f1460 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:27:29.086 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:29.084+0000 7fb88551e700 1 -- 172.21.15.66:0/1820720930 shutdown_connections 2023-11-17T05:27:29.086 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:29.084+0000 7fb88551e700 1 -- 172.21.15.66:0/1820720930 wait complete. 2023-11-17T05:27:29.086 INFO:teuthology.orchestra.run.smithi066.stderr:dumped monmap epoch 1 2023-11-17T05:27:29.988 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:27:29 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:27:29.084100+0000 mon.smithi040 (mon.0) 203 : audit [DBG] from='client.? 172.21.15.66:0/1820720930' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-11-17T05:27:30.471 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2023-11-17T05:27:30.471 DEBUG:teuthology.orchestra.run.smithi066:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.4 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ed6470e2-8508-11ee-8dbe-212e2dc638e7 -- ceph mon dump -f json 2023-11-17T05:27:30.989 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:27:30 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:27:29.217013+0000 mgr.smithi040.ogcprl (mgr.14164) 195 : cluster [DBG] pgmap v183: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:27:31.640 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:31.639+0000 7f004a945700 1 -- 172.21.15.66:0/1265900973 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f00440f5810 msgr2=0x7f00440f5c30 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:27:31.640 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:31.639+0000 7f004a945700 1 --2- 172.21.15.66:0/1265900973 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f00440f5810 0x7f00440f5c30 secure :-1 s=READY pgs=219 cs=0 l=1 rev1=1 rx=0x7f0040009fb0 tx=0x7f0040009b30).stop 2023-11-17T05:27:31.640 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:31.639+0000 7f004a945700 1 -- 172.21.15.66:0/1265900973 shutdown_connections 2023-11-17T05:27:31.641 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:31.639+0000 7f004a945700 1 --2- 172.21.15.66:0/1265900973 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f00440f5810 0x7f00440f5c30 unknown :-1 s=CLOSED pgs=219 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:27:31.641 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:31.639+0000 7f004a945700 1 -- 172.21.15.66:0/1265900973 >> 172.21.15.66:0/1265900973 conn(0x7f00440f0120 msgr2=0x7f00440f2540 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:27:31.641 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:31.639+0000 7f004a945700 1 -- 172.21.15.66:0/1265900973 shutdown_connections 2023-11-17T05:27:31.641 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:31.639+0000 7f004a945700 1 -- 172.21.15.66:0/1265900973 wait complete. 2023-11-17T05:27:31.642 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:31.639+0000 7f004a945700 1 Processor -- start 2023-11-17T05:27:31.642 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:31.639+0000 7f004a945700 1 -- start start 2023-11-17T05:27:31.642 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:31.639+0000 7f004a945700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f00440f5810 0x7f00441086f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:27:31.642 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:31.640+0000 7f004a945700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f0044108be0 con 0x7f00440f5810 2023-11-17T05:27:31.642 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:31.640+0000 7f0049943700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f00440f5810 0x7f00441086f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:27:31.642 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:31.640+0000 7f0049943700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f00440f5810 0x7f00441086f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.66:47946/0 (socket says 172.21.15.66:47946) 2023-11-17T05:27:31.643 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:31.640+0000 7f0049943700 1 -- 172.21.15.66:0/2227024577 learned_addr learned my addr 172.21.15.66:0/2227024577 (peer_addr_for_me v2:172.21.15.66:0/0) 2023-11-17T05:27:31.643 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:31.640+0000 7f0049943700 1 -- 172.21.15.66:0/2227024577 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f0040005040 con 0x7f00440f5810 2023-11-17T05:27:31.643 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:31.640+0000 7f0049943700 1 --2- 172.21.15.66:0/2227024577 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f00440f5810 0x7f00441086f0 secure :-1 s=READY pgs=220 cs=0 l=1 rev1=1 rx=0x7f0040004dd0 tx=0x7f0040009b30).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:27:31.643 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:31.641+0000 7f003a7fc700 1 -- 172.21.15.66:0/2227024577 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f0040018070 con 0x7f00440f5810 2023-11-17T05:27:31.643 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:31.641+0000 7f004a945700 1 -- 172.21.15.66:0/2227024577 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f0044108de0 con 0x7f00440f5810 2023-11-17T05:27:31.643 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:31.641+0000 7f004a945700 1 -- 172.21.15.66:0/2227024577 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f0044109270 con 0x7f00440f5810 2023-11-17T05:27:31.643 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:31.641+0000 7f003a7fc700 1 -- 172.21.15.66:0/2227024577 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(25 keys) v1 ==== 1074+0+0 (secure 0 0 0) 0x7f0040013410 con 0x7f00440f5810 2023-11-17T05:27:31.643 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:31.641+0000 7f003a7fc700 1 -- 172.21.15.66:0/2227024577 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f004001c3c0 con 0x7f00440f5810 2023-11-17T05:27:31.644 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:31.641+0000 7f003a7fc700 1 -- 172.21.15.66:0/2227024577 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 17) v1 ==== 36269+0+0 (secure 0 0 0) 0x7f004001d420 con 0x7f00440f5810 2023-11-17T05:27:31.644 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:31.641+0000 7f004a945700 1 -- 172.21.15.66:0/2227024577 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f0044045ad0 con 0x7f00440f5810 2023-11-17T05:27:31.644 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:31.642+0000 7f003a7fc700 1 --2- 172.21.15.66:0/2227024577 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f003002ff60 0x7f0030032420 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:27:31.644 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:31.642+0000 7f003a7fc700 1 -- 172.21.15.66:0/2227024577 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(5..5 src has 1..5) v4 ==== 1270+0+0 (secure 0 0 0) 0x7f0040041460 con 0x7f00440f5810 2023-11-17T05:27:31.644 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:31.642+0000 7f0049142700 1 --2- 172.21.15.66:0/2227024577 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f003002ff60 0x7f0030032420 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:27:31.644 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:31.642+0000 7f0049142700 1 --2- 172.21.15.66:0/2227024577 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f003002ff60 0x7f0030032420 secure :-1 s=READY pgs=75 cs=0 l=1 rev1=1 rx=0x7f00340096a0 tx=0x7f0034006b40).ready entity=mgr.14164 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:27:31.648 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:31.647+0000 7f003a7fc700 1 -- 172.21.15.66:0/2227024577 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+152212 (secure 0 0 0) 0x7f0040013820 con 0x7f00440f5810 2023-11-17T05:27:31.867 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:31.866+0000 7f004a945700 1 -- 172.21.15.66:0/2227024577 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "mon dump", "format": "json"} v 0) v1 -- 0x7f004410a120 con 0x7f00440f5810 2023-11-17T05:27:31.867 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:31.866+0000 7f003a7fc700 1 -- 172.21.15.66:0/2227024577 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mon_command_ack([{"prefix": "mon dump", "format": "json"}]=0 dumped monmap epoch 1 v1) v1 ==== 95+0+690 (secure 0 0 0) 0x7f004001b370 con 0x7f00440f5810 2023-11-17T05:27:31.868 INFO:teuthology.orchestra.run.smithi066.stdout: 2023-11-17T05:27:31.868 INFO:teuthology.orchestra.run.smithi066.stdout:{"epoch":1,"fsid":"ed6470e2-8508-11ee-8dbe-212e2dc638e7","modified":"2023-11-17T05:20:13.934038Z","created":"2023-11-17T05:20:13.934038Z","min_mon_release":16,"min_mon_release_name":"pacific","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging"],"optional":[]},"mons":[{"rank":0,"name":"smithi040","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:3300","nonce":0},{"type":"v1","addr":"172.21.15.40:6789","nonce":0}]},"addr":"172.21.15.40:6789/0","public_addr":"172.21.15.40:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2023-11-17T05:27:31.869 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:31.868+0000 7f004a945700 1 -- 172.21.15.66:0/2227024577 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f003002ff60 msgr2=0x7f0030032420 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:27:31.869 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:31.868+0000 7f004a945700 1 --2- 172.21.15.66:0/2227024577 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f003002ff60 0x7f0030032420 secure :-1 s=READY pgs=75 cs=0 l=1 rev1=1 rx=0x7f00340096a0 tx=0x7f0034006b40).stop 2023-11-17T05:27:31.869 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:31.868+0000 7f004a945700 1 -- 172.21.15.66:0/2227024577 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f00440f5810 msgr2=0x7f00441086f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:27:31.869 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:31.868+0000 7f004a945700 1 --2- 172.21.15.66:0/2227024577 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f00440f5810 0x7f00441086f0 secure :-1 s=READY pgs=220 cs=0 l=1 rev1=1 rx=0x7f0040004dd0 tx=0x7f0040009b30).stop 2023-11-17T05:27:31.870 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:31.868+0000 7f004a945700 1 -- 172.21.15.66:0/2227024577 shutdown_connections 2023-11-17T05:27:31.870 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:31.868+0000 7f004a945700 1 --2- 172.21.15.66:0/2227024577 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f003002ff60 0x7f0030032420 unknown :-1 s=CLOSED pgs=75 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:27:31.870 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:31.868+0000 7f004a945700 1 --2- 172.21.15.66:0/2227024577 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f00440f5810 0x7f00441086f0 unknown :-1 s=CLOSED pgs=220 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:27:31.870 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:31.868+0000 7f004a945700 1 -- 172.21.15.66:0/2227024577 >> 172.21.15.66:0/2227024577 conn(0x7f00440f0120 msgr2=0x7f00440f4dd0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:27:31.870 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:31.868+0000 7f004a945700 1 -- 172.21.15.66:0/2227024577 shutdown_connections 2023-11-17T05:27:31.870 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:31.868+0000 7f004a945700 1 -- 172.21.15.66:0/2227024577 wait complete. 2023-11-17T05:27:31.870 INFO:teuthology.orchestra.run.smithi066.stderr:dumped monmap epoch 1 2023-11-17T05:27:32.989 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:27:32 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:27:31.217385+0000 mgr.smithi040.ogcprl (mgr.14164) 196 : cluster [DBG] pgmap v184: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:27:32.989 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:27:32 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:27:31.867914+0000 mon.smithi040 (mon.0) 204 : audit [DBG] from='client.? 172.21.15.66:0/2227024577' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-11-17T05:27:33.289 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2023-11-17T05:27:33.289 DEBUG:teuthology.orchestra.run.smithi066:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.4 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ed6470e2-8508-11ee-8dbe-212e2dc638e7 -- ceph mon dump -f json 2023-11-17T05:27:34.460 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:34.458+0000 7f8b1f59e700 1 -- 172.21.15.66:0/2058236875 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f8b200f62a0 msgr2=0x7f8b200f66c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:27:34.460 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:34.459+0000 7f8b1f59e700 1 --2- 172.21.15.66:0/2058236875 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f8b200f62a0 0x7f8b200f66c0 secure :-1 s=READY pgs=221 cs=0 l=1 rev1=1 rx=0x7f8b10009fb0 tx=0x7f8b10009b30).stop 2023-11-17T05:27:34.460 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:34.459+0000 7f8b1f59e700 1 -- 172.21.15.66:0/2058236875 shutdown_connections 2023-11-17T05:27:34.460 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:34.459+0000 7f8b1f59e700 1 --2- 172.21.15.66:0/2058236875 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f8b200f62a0 0x7f8b200f66c0 unknown :-1 s=CLOSED pgs=221 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:27:34.460 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:34.459+0000 7f8b1f59e700 1 -- 172.21.15.66:0/2058236875 >> 172.21.15.66:0/2058236875 conn(0x7f8b200f1aa0 msgr2=0x7f8b200f3ec0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:27:34.460 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:34.459+0000 7f8b1f59e700 1 -- 172.21.15.66:0/2058236875 shutdown_connections 2023-11-17T05:27:34.461 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:34.459+0000 7f8b1f59e700 1 -- 172.21.15.66:0/2058236875 wait complete. 2023-11-17T05:27:34.462 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:34.459+0000 7f8b1f59e700 1 Processor -- start 2023-11-17T05:27:34.462 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:34.459+0000 7f8b1f59e700 1 -- start start 2023-11-17T05:27:34.462 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:34.459+0000 7f8b1f59e700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f8b200f62a0 0x7f8b201064b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:27:34.462 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:34.459+0000 7f8b1f59e700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f8b201069a0 con 0x7f8b200f62a0 2023-11-17T05:27:34.462 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:34.460+0000 7f8b1e59c700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f8b200f62a0 0x7f8b201064b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:27:34.463 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:34.460+0000 7f8b1e59c700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f8b200f62a0 0x7f8b201064b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.66:37772/0 (socket says 172.21.15.66:37772) 2023-11-17T05:27:34.463 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:34.460+0000 7f8b1e59c700 1 -- 172.21.15.66:0/1094355145 learned_addr learned my addr 172.21.15.66:0/1094355145 (peer_addr_for_me v2:172.21.15.66:0/0) 2023-11-17T05:27:34.463 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:34.460+0000 7f8b1e59c700 1 -- 172.21.15.66:0/1094355145 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f8b10005040 con 0x7f8b200f62a0 2023-11-17T05:27:34.463 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:34.460+0000 7f8b1e59c700 1 --2- 172.21.15.66:0/1094355145 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f8b200f62a0 0x7f8b201064b0 secure :-1 s=READY pgs=222 cs=0 l=1 rev1=1 rx=0x7f8b10013510 tx=0x7f8b1000b040).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:27:34.463 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:34.460+0000 7f8b0effd700 1 -- 172.21.15.66:0/1094355145 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f8b10009320 con 0x7f8b200f62a0 2023-11-17T05:27:34.463 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:34.460+0000 7f8b1f59e700 1 -- 172.21.15.66:0/1094355145 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f8b20106ba0 con 0x7f8b200f62a0 2023-11-17T05:27:34.463 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:34.460+0000 7f8b0effd700 1 -- 172.21.15.66:0/1094355145 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(25 keys) v1 ==== 1074+0+0 (secure 0 0 0) 0x7f8b10013f00 con 0x7f8b200f62a0 2023-11-17T05:27:34.463 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:34.460+0000 7f8b0effd700 1 -- 172.21.15.66:0/1094355145 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f8b10014dc0 con 0x7f8b200f62a0 2023-11-17T05:27:34.463 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:34.460+0000 7f8b1f59e700 1 -- 172.21.15.66:0/1094355145 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f8b20107030 con 0x7f8b200f62a0 2023-11-17T05:27:34.463 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:34.461+0000 7f8b0effd700 1 -- 172.21.15.66:0/1094355145 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 17) v1 ==== 36269+0+0 (secure 0 0 0) 0x7f8b10014420 con 0x7f8b200f62a0 2023-11-17T05:27:34.463 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:34.461+0000 7f8b1f59e700 1 -- 172.21.15.66:0/1094355145 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f8b20045ad0 con 0x7f8b200f62a0 2023-11-17T05:27:34.464 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:34.461+0000 7f8b0effd700 1 --2- 172.21.15.66:0/1094355145 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f8b0802ff60 0x7f8b08032420 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:27:34.464 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:34.461+0000 7f8b0effd700 1 -- 172.21.15.66:0/1094355145 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(5..5 src has 1..5) v4 ==== 1270+0+0 (secure 0 0 0) 0x7f8b10041030 con 0x7f8b200f62a0 2023-11-17T05:27:34.464 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:34.461+0000 7f8b1dd9b700 1 --2- 172.21.15.66:0/1094355145 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f8b0802ff60 0x7f8b08032420 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:27:34.464 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:34.462+0000 7f8b1dd9b700 1 --2- 172.21.15.66:0/1094355145 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f8b0802ff60 0x7f8b08032420 secure :-1 s=READY pgs=76 cs=0 l=1 rev1=1 rx=0x7f8b140096a0 tx=0x7f8b14006b40).ready entity=mgr.14164 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:27:34.468 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:34.466+0000 7f8b0effd700 1 -- 172.21.15.66:0/1094355145 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+152212 (secure 0 0 0) 0x7f8b100095f0 con 0x7f8b200f62a0 2023-11-17T05:27:34.687 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:34.685+0000 7f8b1f59e700 1 -- 172.21.15.66:0/1094355145 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "mon dump", "format": "json"} v 0) v1 -- 0x7f8b20107ee0 con 0x7f8b200f62a0 2023-11-17T05:27:34.687 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:34.686+0000 7f8b0effd700 1 -- 172.21.15.66:0/1094355145 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mon_command_ack([{"prefix": "mon dump", "format": "json"}]=0 dumped monmap epoch 1 v1) v1 ==== 95+0+690 (secure 0 0 0) 0x7f8b1001cb10 con 0x7f8b200f62a0 2023-11-17T05:27:34.688 INFO:teuthology.orchestra.run.smithi066.stdout: 2023-11-17T05:27:34.688 INFO:teuthology.orchestra.run.smithi066.stdout:{"epoch":1,"fsid":"ed6470e2-8508-11ee-8dbe-212e2dc638e7","modified":"2023-11-17T05:20:13.934038Z","created":"2023-11-17T05:20:13.934038Z","min_mon_release":16,"min_mon_release_name":"pacific","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging"],"optional":[]},"mons":[{"rank":0,"name":"smithi040","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:3300","nonce":0},{"type":"v1","addr":"172.21.15.40:6789","nonce":0}]},"addr":"172.21.15.40:6789/0","public_addr":"172.21.15.40:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2023-11-17T05:27:34.689 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:34.688+0000 7f8b1f59e700 1 -- 172.21.15.66:0/1094355145 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f8b0802ff60 msgr2=0x7f8b08032420 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:27:34.689 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:34.688+0000 7f8b1f59e700 1 --2- 172.21.15.66:0/1094355145 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f8b0802ff60 0x7f8b08032420 secure :-1 s=READY pgs=76 cs=0 l=1 rev1=1 rx=0x7f8b140096a0 tx=0x7f8b14006b40).stop 2023-11-17T05:27:34.689 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:34.688+0000 7f8b1f59e700 1 -- 172.21.15.66:0/1094355145 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f8b200f62a0 msgr2=0x7f8b201064b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:27:34.689 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:34.688+0000 7f8b1f59e700 1 --2- 172.21.15.66:0/1094355145 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f8b200f62a0 0x7f8b201064b0 secure :-1 s=READY pgs=222 cs=0 l=1 rev1=1 rx=0x7f8b10013510 tx=0x7f8b1000b040).stop 2023-11-17T05:27:34.689 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:34.688+0000 7f8b1f59e700 1 -- 172.21.15.66:0/1094355145 shutdown_connections 2023-11-17T05:27:34.690 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:34.688+0000 7f8b1f59e700 1 --2- 172.21.15.66:0/1094355145 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f8b0802ff60 0x7f8b08032420 unknown :-1 s=CLOSED pgs=76 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:27:34.690 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:34.688+0000 7f8b1f59e700 1 --2- 172.21.15.66:0/1094355145 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f8b200f62a0 0x7f8b201064b0 unknown :-1 s=CLOSED pgs=222 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:27:34.690 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:34.688+0000 7f8b1f59e700 1 -- 172.21.15.66:0/1094355145 >> 172.21.15.66:0/1094355145 conn(0x7f8b200f1aa0 msgr2=0x7f8b200f2510 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:27:34.690 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:34.688+0000 7f8b1f59e700 1 -- 172.21.15.66:0/1094355145 shutdown_connections 2023-11-17T05:27:34.690 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:34.688+0000 7f8b1f59e700 1 -- 172.21.15.66:0/1094355145 wait complete. 2023-11-17T05:27:34.690 INFO:teuthology.orchestra.run.smithi066.stderr:dumped monmap epoch 1 2023-11-17T05:27:34.988 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:27:34 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:27:33.217854+0000 mgr.smithi040.ogcprl (mgr.14164) 197 : cluster [DBG] pgmap v185: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:27:35.988 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:27:35 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:27:34.687916+0000 mon.smithi040 (mon.0) 205 : audit [DBG] from='client.? 172.21.15.66:0/1094355145' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-11-17T05:27:36.071 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2023-11-17T05:27:36.071 DEBUG:teuthology.orchestra.run.smithi066:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.4 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ed6470e2-8508-11ee-8dbe-212e2dc638e7 -- ceph mon dump -f json 2023-11-17T05:27:36.988 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:27:36 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:27:35.218191+0000 mgr.smithi040.ogcprl (mgr.14164) 198 : cluster [DBG] pgmap v186: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:27:37.256 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:37.246+0000 7fcbfd610700 1 -- 172.21.15.66:0/3058523142 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fcbf8060280 msgr2=0x7fcbf80f90f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:27:37.257 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:37.246+0000 7fcbfd610700 1 --2- 172.21.15.66:0/3058523142 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fcbf8060280 0x7fcbf80f90f0 secure :-1 s=READY pgs=223 cs=0 l=1 rev1=1 rx=0x7fcbec009fb0 tx=0x7fcbec009b30).stop 2023-11-17T05:27:37.262 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:37.246+0000 7fcbfd610700 1 -- 172.21.15.66:0/3058523142 shutdown_connections 2023-11-17T05:27:37.262 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:37.246+0000 7fcbfd610700 1 --2- 172.21.15.66:0/3058523142 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fcbf8060280 0x7fcbf80f90f0 unknown :-1 s=CLOSED pgs=223 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:27:37.262 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:37.246+0000 7fcbfd610700 1 -- 172.21.15.66:0/3058523142 >> 172.21.15.66:0/3058523142 conn(0x7fcbf80f0b70 msgr2=0x7fcbf80f2f90 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:27:37.262 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:37.246+0000 7fcbfd610700 1 -- 172.21.15.66:0/3058523142 shutdown_connections 2023-11-17T05:27:37.262 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:37.246+0000 7fcbfd610700 1 -- 172.21.15.66:0/3058523142 wait complete. 2023-11-17T05:27:37.262 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:37.247+0000 7fcbfd610700 1 Processor -- start 2023-11-17T05:27:37.263 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:37.247+0000 7fcbfd610700 1 -- start start 2023-11-17T05:27:37.263 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:37.247+0000 7fcbfd610700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fcbf81089c0 0x7fcbf8108de0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:27:37.263 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:37.247+0000 7fcbfd610700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fcbf81092d0 con 0x7fcbf81089c0 2023-11-17T05:27:37.263 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:37.248+0000 7fcbf6ffd700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fcbf81089c0 0x7fcbf8108de0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:27:37.263 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:37.248+0000 7fcbf6ffd700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fcbf81089c0 0x7fcbf8108de0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.66:37786/0 (socket says 172.21.15.66:37786) 2023-11-17T05:27:37.263 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:37.248+0000 7fcbf6ffd700 1 -- 172.21.15.66:0/1756202529 learned_addr learned my addr 172.21.15.66:0/1756202529 (peer_addr_for_me v2:172.21.15.66:0/0) 2023-11-17T05:27:37.263 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:37.248+0000 7fcbf6ffd700 1 -- 172.21.15.66:0/1756202529 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fcbec005040 con 0x7fcbf81089c0 2023-11-17T05:27:37.263 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:37.248+0000 7fcbf6ffd700 1 --2- 172.21.15.66:0/1756202529 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fcbf81089c0 0x7fcbf8108de0 secure :-1 s=READY pgs=224 cs=0 l=1 rev1=1 rx=0x7fcbec009f80 tx=0x7fcbec009b30).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:27:37.263 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:37.248+0000 7fcbdf7fe700 1 -- 172.21.15.66:0/1756202529 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7fcbec0136c0 con 0x7fcbf81089c0 2023-11-17T05:27:37.263 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:37.248+0000 7fcbfd610700 1 -- 172.21.15.66:0/1756202529 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fcbf81094d0 con 0x7fcbf81089c0 2023-11-17T05:27:37.263 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:37.248+0000 7fcbfd610700 1 -- 172.21.15.66:0/1756202529 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fcbf810a020 con 0x7fcbf81089c0 2023-11-17T05:27:37.263 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:37.248+0000 7fcbdf7fe700 1 -- 172.21.15.66:0/1756202529 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(25 keys) v1 ==== 1074+0+0 (secure 0 0 0) 0x7fcbec0095f0 con 0x7fcbf81089c0 2023-11-17T05:27:37.263 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:37.248+0000 7fcbdf7fe700 1 -- 172.21.15.66:0/1756202529 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7fcbec01d3a0 con 0x7fcbf81089c0 2023-11-17T05:27:37.263 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:37.249+0000 7fcbdf7fe700 1 -- 172.21.15.66:0/1756202529 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 17) v1 ==== 36269+0+0 (secure 0 0 0) 0x7fcbec025060 con 0x7fcbf81089c0 2023-11-17T05:27:37.264 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:37.249+0000 7fcbdf7fe700 1 --2- 172.21.15.66:0/1756202529 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fcbe002ff60 0x7fcbe0032420 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:27:37.264 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:37.249+0000 7fcbdf7fe700 1 -- 172.21.15.66:0/1756202529 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(5..5 src has 1..5) v4 ==== 1270+0+0 (secure 0 0 0) 0x7fcbec040b20 con 0x7fcbf81089c0 2023-11-17T05:27:37.264 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:37.249+0000 7fcbfd610700 1 -- 172.21.15.66:0/1756202529 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fcbf8059100 con 0x7fcbf81089c0 2023-11-17T05:27:37.264 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:37.250+0000 7fcbf67fc700 1 --2- 172.21.15.66:0/1756202529 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fcbe002ff60 0x7fcbe0032420 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:27:37.264 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:37.250+0000 7fcbf67fc700 1 --2- 172.21.15.66:0/1756202529 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fcbe002ff60 0x7fcbe0032420 secure :-1 s=READY pgs=77 cs=0 l=1 rev1=1 rx=0x7fcbe80096a0 tx=0x7fcbe8006b40).ready entity=mgr.14164 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:27:37.265 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:37.254+0000 7fcbdf7fe700 1 -- 172.21.15.66:0/1756202529 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+152212 (secure 0 0 0) 0x7fcbec01e440 con 0x7fcbf81089c0 2023-11-17T05:27:37.478 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:37.476+0000 7fcbfd610700 1 -- 172.21.15.66:0/1756202529 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "mon dump", "format": "json"} v 0) v1 -- 0x7fcbf810a680 con 0x7fcbf81089c0 2023-11-17T05:27:37.478 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:37.477+0000 7fcbdf7fe700 1 -- 172.21.15.66:0/1756202529 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mon_command_ack([{"prefix": "mon dump", "format": "json"}]=0 dumped monmap epoch 1 v1) v1 ==== 95+0+690 (secure 0 0 0) 0x7fcbec0252f0 con 0x7fcbf81089c0 2023-11-17T05:27:37.478 INFO:teuthology.orchestra.run.smithi066.stdout: 2023-11-17T05:27:37.479 INFO:teuthology.orchestra.run.smithi066.stdout:{"epoch":1,"fsid":"ed6470e2-8508-11ee-8dbe-212e2dc638e7","modified":"2023-11-17T05:20:13.934038Z","created":"2023-11-17T05:20:13.934038Z","min_mon_release":16,"min_mon_release_name":"pacific","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging"],"optional":[]},"mons":[{"rank":0,"name":"smithi040","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:3300","nonce":0},{"type":"v1","addr":"172.21.15.40:6789","nonce":0}]},"addr":"172.21.15.40:6789/0","public_addr":"172.21.15.40:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2023-11-17T05:27:37.480 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:37.479+0000 7fcbfd610700 1 -- 172.21.15.66:0/1756202529 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fcbe002ff60 msgr2=0x7fcbe0032420 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:27:37.480 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:37.479+0000 7fcbfd610700 1 --2- 172.21.15.66:0/1756202529 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fcbe002ff60 0x7fcbe0032420 secure :-1 s=READY pgs=77 cs=0 l=1 rev1=1 rx=0x7fcbe80096a0 tx=0x7fcbe8006b40).stop 2023-11-17T05:27:37.480 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:37.479+0000 7fcbfd610700 1 -- 172.21.15.66:0/1756202529 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fcbf81089c0 msgr2=0x7fcbf8108de0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:27:37.480 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:37.479+0000 7fcbfd610700 1 --2- 172.21.15.66:0/1756202529 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fcbf81089c0 0x7fcbf8108de0 secure :-1 s=READY pgs=224 cs=0 l=1 rev1=1 rx=0x7fcbec009f80 tx=0x7fcbec009b30).stop 2023-11-17T05:27:37.481 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:37.479+0000 7fcbfd610700 1 -- 172.21.15.66:0/1756202529 shutdown_connections 2023-11-17T05:27:37.481 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:37.479+0000 7fcbfd610700 1 --2- 172.21.15.66:0/1756202529 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fcbe002ff60 0x7fcbe0032420 unknown :-1 s=CLOSED pgs=77 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:27:37.481 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:37.479+0000 7fcbfd610700 1 --2- 172.21.15.66:0/1756202529 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fcbf81089c0 0x7fcbf8108de0 unknown :-1 s=CLOSED pgs=224 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:27:37.481 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:37.479+0000 7fcbfd610700 1 -- 172.21.15.66:0/1756202529 >> 172.21.15.66:0/1756202529 conn(0x7fcbf80f0b70 msgr2=0x7fcbf80f16c0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:27:37.481 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:37.479+0000 7fcbfd610700 1 -- 172.21.15.66:0/1756202529 shutdown_connections 2023-11-17T05:27:37.481 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:37.479+0000 7fcbfd610700 1 -- 172.21.15.66:0/1756202529 wait complete. 2023-11-17T05:27:37.481 INFO:teuthology.orchestra.run.smithi066.stderr:dumped monmap epoch 1 2023-11-17T05:27:37.989 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:27:37 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:27:37.218675+0000 mgr.smithi040.ogcprl (mgr.14164) 199 : cluster [DBG] pgmap v187: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:27:37.989 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:27:37 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:27:37.478633+0000 mon.smithi040 (mon.0) 206 : audit [DBG] from='client.? 172.21.15.66:0/1756202529' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-11-17T05:27:38.860 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2023-11-17T05:27:38.860 DEBUG:teuthology.orchestra.run.smithi066:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.4 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ed6470e2-8508-11ee-8dbe-212e2dc638e7 -- ceph mon dump -f json 2023-11-17T05:27:40.125 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:40.104+0000 7f900ef0b700 1 -- 172.21.15.66:0/1360693273 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f90080f5410 msgr2=0x7f90080f5830 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:27:40.125 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:40.104+0000 7f900ef0b700 1 --2- 172.21.15.66:0/1360693273 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f90080f5410 0x7f90080f5830 secure :-1 s=READY pgs=225 cs=0 l=1 rev1=1 rx=0x7f8ffc009fb0 tx=0x7f8ffc009b30).stop 2023-11-17T05:27:40.125 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:40.104+0000 7f900ef0b700 1 -- 172.21.15.66:0/1360693273 shutdown_connections 2023-11-17T05:27:40.125 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:40.104+0000 7f900ef0b700 1 --2- 172.21.15.66:0/1360693273 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f90080f5410 0x7f90080f5830 unknown :-1 s=CLOSED pgs=225 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:27:40.125 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:40.104+0000 7f900ef0b700 1 -- 172.21.15.66:0/1360693273 >> 172.21.15.66:0/1360693273 conn(0x7f90080f0b70 msgr2=0x7f90080f2f90 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:27:40.125 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:40.104+0000 7f900ef0b700 1 -- 172.21.15.66:0/1360693273 shutdown_connections 2023-11-17T05:27:40.125 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:40.104+0000 7f900ef0b700 1 -- 172.21.15.66:0/1360693273 wait complete. 2023-11-17T05:27:40.125 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:40.105+0000 7f900ef0b700 1 Processor -- start 2023-11-17T05:27:40.125 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:40.105+0000 7f900ef0b700 1 -- start start 2023-11-17T05:27:40.125 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:40.105+0000 7f900ef0b700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f90080f5410 0x7f9008105630 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:27:40.126 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:40.105+0000 7f900ef0b700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f9008105b20 con 0x7f90080f5410 2023-11-17T05:27:40.126 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:40.105+0000 7f900cca7700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f90080f5410 0x7f9008105630 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:27:40.126 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:40.105+0000 7f900cca7700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f90080f5410 0x7f9008105630 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.66:37802/0 (socket says 172.21.15.66:37802) 2023-11-17T05:27:40.126 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:40.105+0000 7f900cca7700 1 -- 172.21.15.66:0/3097892619 learned_addr learned my addr 172.21.15.66:0/3097892619 (peer_addr_for_me v2:172.21.15.66:0/0) 2023-11-17T05:27:40.126 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:40.106+0000 7f900cca7700 1 -- 172.21.15.66:0/3097892619 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f8ffc005040 con 0x7f90080f5410 2023-11-17T05:27:40.126 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:40.106+0000 7f900cca7700 1 --2- 172.21.15.66:0/3097892619 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f90080f5410 0x7f9008105630 secure :-1 s=READY pgs=226 cs=0 l=1 rev1=1 rx=0x7f8ffc013510 tx=0x7f8ffc00b040).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:27:40.126 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:40.106+0000 7f90057fa700 1 -- 172.21.15.66:0/3097892619 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f8ffc009320 con 0x7f90080f5410 2023-11-17T05:27:40.126 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:40.106+0000 7f900ef0b700 1 -- 172.21.15.66:0/3097892619 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f9008105d20 con 0x7f90080f5410 2023-11-17T05:27:40.126 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:40.106+0000 7f900ef0b700 1 -- 172.21.15.66:0/3097892619 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f90081061b0 con 0x7f90080f5410 2023-11-17T05:27:40.126 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:40.106+0000 7f90057fa700 1 -- 172.21.15.66:0/3097892619 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(25 keys) v1 ==== 1074+0+0 (secure 0 0 0) 0x7f8ffc013f00 con 0x7f90080f5410 2023-11-17T05:27:40.126 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:40.106+0000 7f90057fa700 1 -- 172.21.15.66:0/3097892619 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f8ffc014d60 con 0x7f90080f5410 2023-11-17T05:27:40.126 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:40.107+0000 7f90057fa700 1 -- 172.21.15.66:0/3097892619 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 17) v1 ==== 36269+0+0 (secure 0 0 0) 0x7f8ffc0095f0 con 0x7f90080f5410 2023-11-17T05:27:40.127 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:40.107+0000 7f900ef0b700 1 -- 172.21.15.66:0/3097892619 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f9008045ad0 con 0x7f90080f5410 2023-11-17T05:27:40.127 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:40.107+0000 7f90057fa700 1 --2- 172.21.15.66:0/3097892619 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f8ff802ff60 0x7f8ff8032420 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:27:40.127 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:40.107+0000 7f90057fa700 1 -- 172.21.15.66:0/3097892619 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(5..5 src has 1..5) v4 ==== 1270+0+0 (secure 0 0 0) 0x7f8ffc042e60 con 0x7f90080f5410 2023-11-17T05:27:40.127 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:40.108+0000 7f9007fff700 1 --2- 172.21.15.66:0/3097892619 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f8ff802ff60 0x7f8ff8032420 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:27:40.128 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:40.108+0000 7f9007fff700 1 --2- 172.21.15.66:0/3097892619 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f8ff802ff60 0x7f8ff8032420 secure :-1 s=READY pgs=78 cs=0 l=1 rev1=1 rx=0x7f8ff40096a0 tx=0x7f8ff4006b40).ready entity=mgr.14164 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:27:40.129 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:40.112+0000 7f90057fa700 1 -- 172.21.15.66:0/3097892619 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+152212 (secure 0 0 0) 0x7f8ffc014420 con 0x7f90080f5410 2023-11-17T05:27:40.329 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:40.328+0000 7f900ef0b700 1 -- 172.21.15.66:0/3097892619 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "mon dump", "format": "json"} v 0) v1 -- 0x7f90080fae60 con 0x7f90080f5410 2023-11-17T05:27:40.330 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:40.330+0000 7f90057fa700 1 -- 172.21.15.66:0/3097892619 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mon_command_ack([{"prefix": "mon dump", "format": "json"}]=0 dumped monmap epoch 1 v1) v1 ==== 95+0+690 (secure 0 0 0) 0x7f8ffc0127a0 con 0x7f90080f5410 2023-11-17T05:27:40.330 INFO:teuthology.orchestra.run.smithi066.stdout: 2023-11-17T05:27:40.330 INFO:teuthology.orchestra.run.smithi066.stdout:{"epoch":1,"fsid":"ed6470e2-8508-11ee-8dbe-212e2dc638e7","modified":"2023-11-17T05:20:13.934038Z","created":"2023-11-17T05:20:13.934038Z","min_mon_release":16,"min_mon_release_name":"pacific","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging"],"optional":[]},"mons":[{"rank":0,"name":"smithi040","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:3300","nonce":0},{"type":"v1","addr":"172.21.15.40:6789","nonce":0}]},"addr":"172.21.15.40:6789/0","public_addr":"172.21.15.40:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2023-11-17T05:27:40.332 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:40.330+0000 7f900ef0b700 1 -- 172.21.15.66:0/3097892619 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f8ff802ff60 msgr2=0x7f8ff8032420 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:27:40.332 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:40.330+0000 7f900ef0b700 1 --2- 172.21.15.66:0/3097892619 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f8ff802ff60 0x7f8ff8032420 secure :-1 s=READY pgs=78 cs=0 l=1 rev1=1 rx=0x7f8ff40096a0 tx=0x7f8ff4006b40).stop 2023-11-17T05:27:40.332 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:40.330+0000 7f900ef0b700 1 -- 172.21.15.66:0/3097892619 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f90080f5410 msgr2=0x7f9008105630 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:27:40.332 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:40.330+0000 7f900ef0b700 1 --2- 172.21.15.66:0/3097892619 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f90080f5410 0x7f9008105630 secure :-1 s=READY pgs=226 cs=0 l=1 rev1=1 rx=0x7f8ffc013510 tx=0x7f8ffc00b040).stop 2023-11-17T05:27:40.332 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:40.330+0000 7f900ef0b700 1 -- 172.21.15.66:0/3097892619 shutdown_connections 2023-11-17T05:27:40.332 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:40.330+0000 7f900ef0b700 1 --2- 172.21.15.66:0/3097892619 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f8ff802ff60 0x7f8ff8032420 unknown :-1 s=CLOSED pgs=78 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:27:40.332 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:40.330+0000 7f900ef0b700 1 --2- 172.21.15.66:0/3097892619 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f90080f5410 0x7f9008105630 unknown :-1 s=CLOSED pgs=226 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:27:40.332 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:40.330+0000 7f900ef0b700 1 -- 172.21.15.66:0/3097892619 >> 172.21.15.66:0/3097892619 conn(0x7f90080f0b70 msgr2=0x7f90080f1660 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:27:40.332 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:40.331+0000 7f900ef0b700 1 -- 172.21.15.66:0/3097892619 shutdown_connections 2023-11-17T05:27:40.332 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:40.331+0000 7f900ef0b700 1 -- 172.21.15.66:0/3097892619 wait complete. 2023-11-17T05:27:40.333 INFO:teuthology.orchestra.run.smithi066.stderr:dumped monmap epoch 1 2023-11-17T05:27:40.739 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:27:40 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:27:39.219141+0000 mgr.smithi040.ogcprl (mgr.14164) 200 : cluster [DBG] pgmap v188: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:27:40.739 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:27:40 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:27:40.330461+0000 mon.smithi040 (mon.0) 207 : audit [DBG] from='client.? 172.21.15.66:0/3097892619' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-11-17T05:27:41.741 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2023-11-17T05:27:41.741 DEBUG:teuthology.orchestra.run.smithi066:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.4 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ed6470e2-8508-11ee-8dbe-212e2dc638e7 -- ceph mon dump -f json 2023-11-17T05:27:42.739 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:27:42 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:27:41.219528+0000 mgr.smithi040.ogcprl (mgr.14164) 201 : cluster [DBG] pgmap v189: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:27:43.036 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:43.033+0000 7f6721fbc700 1 -- 172.21.15.66:0/3834528198 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f671c060280 msgr2=0x7f671c0f90f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:27:43.036 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:43.033+0000 7f6721fbc700 1 --2- 172.21.15.66:0/3834528198 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f671c060280 0x7f671c0f90f0 secure :-1 s=READY pgs=227 cs=0 l=1 rev1=1 rx=0x7f670c009fb0 tx=0x7f670c009b30).stop 2023-11-17T05:27:43.036 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:43.033+0000 7f6721fbc700 1 -- 172.21.15.66:0/3834528198 shutdown_connections 2023-11-17T05:27:43.036 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:43.033+0000 7f6721fbc700 1 --2- 172.21.15.66:0/3834528198 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f671c060280 0x7f671c0f90f0 unknown :-1 s=CLOSED pgs=227 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:27:43.036 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:43.033+0000 7f6721fbc700 1 -- 172.21.15.66:0/3834528198 >> 172.21.15.66:0/3834528198 conn(0x7f671c0f0b70 msgr2=0x7f671c0f2f90 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:27:43.036 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:43.033+0000 7f6721fbc700 1 -- 172.21.15.66:0/3834528198 shutdown_connections 2023-11-17T05:27:43.036 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:43.034+0000 7f6721fbc700 1 -- 172.21.15.66:0/3834528198 wait complete. 2023-11-17T05:27:43.036 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:43.034+0000 7f6721fbc700 1 Processor -- start 2023-11-17T05:27:43.036 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:43.034+0000 7f6721fbc700 1 -- start start 2023-11-17T05:27:43.036 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:43.034+0000 7f6721fbc700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f671c060280 0x7f671c0f6b70 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:27:43.037 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:43.034+0000 7f6721fbc700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f671c0f4ff0 con 0x7f671c060280 2023-11-17T05:27:43.037 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:43.035+0000 7f671b7fe700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f671c060280 0x7f671c0f6b70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:27:43.037 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:43.035+0000 7f671b7fe700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f671c060280 0x7f671c0f6b70 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.66:48260/0 (socket says 172.21.15.66:48260) 2023-11-17T05:27:43.037 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:43.035+0000 7f671b7fe700 1 -- 172.21.15.66:0/438240476 learned_addr learned my addr 172.21.15.66:0/438240476 (peer_addr_for_me v2:172.21.15.66:0/0) 2023-11-17T05:27:43.039 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:43.035+0000 7f671b7fe700 1 -- 172.21.15.66:0/438240476 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f670c005040 con 0x7f671c060280 2023-11-17T05:27:43.039 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:43.035+0000 7f671b7fe700 1 --2- 172.21.15.66:0/438240476 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f671c060280 0x7f671c0f6b70 secure :-1 s=READY pgs=228 cs=0 l=1 rev1=1 rx=0x7f670c013510 tx=0x7f670c00b040).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:27:43.039 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:43.035+0000 7f6703fff700 1 -- 172.21.15.66:0/438240476 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f670c009320 con 0x7f671c060280 2023-11-17T05:27:43.039 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:43.035+0000 7f6721fbc700 1 -- 172.21.15.66:0/438240476 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f671c0f5190 con 0x7f671c060280 2023-11-17T05:27:43.039 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:43.035+0000 7f6703fff700 1 -- 172.21.15.66:0/438240476 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(25 keys) v1 ==== 1074+0+0 (secure 0 0 0) 0x7f670c013f00 con 0x7f671c060280 2023-11-17T05:27:43.039 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:43.035+0000 7f6703fff700 1 -- 172.21.15.66:0/438240476 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f670c014dc0 con 0x7f671c060280 2023-11-17T05:27:43.039 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:43.035+0000 7f6721fbc700 1 -- 172.21.15.66:0/438240476 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f671c0f5620 con 0x7f671c060280 2023-11-17T05:27:43.039 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:43.036+0000 7f6703fff700 1 -- 172.21.15.66:0/438240476 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 17) v1 ==== 36269+0+0 (secure 0 0 0) 0x7f670c01d3d0 con 0x7f671c060280 2023-11-17T05:27:43.039 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:43.036+0000 7f6703fff700 1 --2- 172.21.15.66:0/438240476 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f670402ff60 0x7f6704032420 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:27:43.039 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:43.036+0000 7f6703fff700 1 -- 172.21.15.66:0/438240476 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(5..5 src has 1..5) v4 ==== 1270+0+0 (secure 0 0 0) 0x7f670c0095f0 con 0x7f671c060280 2023-11-17T05:27:43.040 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:43.036+0000 7f6721fbc700 1 -- 172.21.15.66:0/438240476 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f671c059100 con 0x7f671c060280 2023-11-17T05:27:43.040 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:43.037+0000 7f671affd700 1 --2- 172.21.15.66:0/438240476 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f670402ff60 0x7f6704032420 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:27:43.040 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:43.037+0000 7f671affd700 1 --2- 172.21.15.66:0/438240476 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f670402ff60 0x7f6704032420 secure :-1 s=READY pgs=79 cs=0 l=1 rev1=1 rx=0x7f67100096a0 tx=0x7f6710006b40).ready entity=mgr.14164 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:27:43.045 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:43.041+0000 7f6703fff700 1 -- 172.21.15.66:0/438240476 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+152212 (secure 0 0 0) 0x7f670c026490 con 0x7f671c060280 2023-11-17T05:27:43.260 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:43.258+0000 7f6721fbc700 1 -- 172.21.15.66:0/438240476 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "mon dump", "format": "json"} v 0) v1 -- 0x7f671c045ad0 con 0x7f671c060280 2023-11-17T05:27:43.260 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:43.259+0000 7f6703fff700 1 -- 172.21.15.66:0/438240476 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mon_command_ack([{"prefix": "mon dump", "format": "json"}]=0 dumped monmap epoch 1 v1) v1 ==== 95+0+690 (secure 0 0 0) 0x7f671c045ad0 con 0x7f671c060280 2023-11-17T05:27:43.261 INFO:teuthology.orchestra.run.smithi066.stdout: 2023-11-17T05:27:43.261 INFO:teuthology.orchestra.run.smithi066.stdout:{"epoch":1,"fsid":"ed6470e2-8508-11ee-8dbe-212e2dc638e7","modified":"2023-11-17T05:20:13.934038Z","created":"2023-11-17T05:20:13.934038Z","min_mon_release":16,"min_mon_release_name":"pacific","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging"],"optional":[]},"mons":[{"rank":0,"name":"smithi040","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:3300","nonce":0},{"type":"v1","addr":"172.21.15.40:6789","nonce":0}]},"addr":"172.21.15.40:6789/0","public_addr":"172.21.15.40:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2023-11-17T05:27:43.262 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:43.260+0000 7f6721fbc700 1 -- 172.21.15.66:0/438240476 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f670402ff60 msgr2=0x7f6704032420 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:27:43.262 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:43.261+0000 7f6721fbc700 1 --2- 172.21.15.66:0/438240476 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f670402ff60 0x7f6704032420 secure :-1 s=READY pgs=79 cs=0 l=1 rev1=1 rx=0x7f67100096a0 tx=0x7f6710006b40).stop 2023-11-17T05:27:43.262 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:43.261+0000 7f6721fbc700 1 -- 172.21.15.66:0/438240476 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f671c060280 msgr2=0x7f671c0f6b70 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:27:43.262 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:43.261+0000 7f6721fbc700 1 --2- 172.21.15.66:0/438240476 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f671c060280 0x7f671c0f6b70 secure :-1 s=READY pgs=228 cs=0 l=1 rev1=1 rx=0x7f670c013510 tx=0x7f670c00b040).stop 2023-11-17T05:27:43.262 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:43.261+0000 7f6721fbc700 1 -- 172.21.15.66:0/438240476 shutdown_connections 2023-11-17T05:27:43.262 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:43.261+0000 7f6721fbc700 1 --2- 172.21.15.66:0/438240476 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f670402ff60 0x7f6704032420 unknown :-1 s=CLOSED pgs=79 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:27:43.262 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:43.261+0000 7f6721fbc700 1 --2- 172.21.15.66:0/438240476 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f671c060280 0x7f671c0f6b70 unknown :-1 s=CLOSED pgs=228 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:27:43.262 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:43.261+0000 7f6721fbc700 1 -- 172.21.15.66:0/438240476 >> 172.21.15.66:0/438240476 conn(0x7f671c0f0b70 msgr2=0x7f671c0f1660 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:27:43.263 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:43.261+0000 7f6721fbc700 1 -- 172.21.15.66:0/438240476 shutdown_connections 2023-11-17T05:27:43.263 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:43.261+0000 7f6721fbc700 1 -- 172.21.15.66:0/438240476 wait complete. 2023-11-17T05:27:43.263 INFO:teuthology.orchestra.run.smithi066.stderr:dumped monmap epoch 1 2023-11-17T05:27:43.739 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:27:43 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:27:43.260755+0000 mon.smithi040 (mon.0) 208 : audit [DBG] from='client.? 172.21.15.66:0/438240476' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-11-17T05:27:44.659 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2023-11-17T05:27:44.659 DEBUG:teuthology.orchestra.run.smithi066:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.4 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ed6470e2-8508-11ee-8dbe-212e2dc638e7 -- ceph mon dump -f json 2023-11-17T05:27:44.739 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:27:44 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:27:43.219851+0000 mgr.smithi040.ogcprl (mgr.14164) 202 : cluster [DBG] pgmap v190: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:27:45.842 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:45.840+0000 7fc22fbea700 1 -- 172.21.15.66:0/1520190361 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fc2280f6510 msgr2=0x7fc2280f6930 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:27:45.843 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:45.840+0000 7fc22fbea700 1 --2- 172.21.15.66:0/1520190361 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fc2280f6510 0x7fc2280f6930 secure :-1 s=READY pgs=229 cs=0 l=1 rev1=1 rx=0x7fc224009fb0 tx=0x7fc224009b30).stop 2023-11-17T05:27:45.843 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:45.840+0000 7fc22fbea700 1 -- 172.21.15.66:0/1520190361 shutdown_connections 2023-11-17T05:27:45.843 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:45.840+0000 7fc22fbea700 1 --2- 172.21.15.66:0/1520190361 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fc2280f6510 0x7fc2280f6930 unknown :-1 s=CLOSED pgs=229 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:27:45.843 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:45.840+0000 7fc22fbea700 1 -- 172.21.15.66:0/1520190361 >> 172.21.15.66:0/1520190361 conn(0x7fc2280f1c70 msgr2=0x7fc2280f4090 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:27:45.843 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:45.840+0000 7fc22fbea700 1 -- 172.21.15.66:0/1520190361 shutdown_connections 2023-11-17T05:27:45.843 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:45.840+0000 7fc22fbea700 1 -- 172.21.15.66:0/1520190361 wait complete. 2023-11-17T05:27:45.843 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:45.841+0000 7fc22fbea700 1 Processor -- start 2023-11-17T05:27:45.843 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:45.841+0000 7fc22fbea700 1 -- start start 2023-11-17T05:27:45.843 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:45.841+0000 7fc22fbea700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fc2280f6510 0x7fc228106730 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:27:45.843 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:45.841+0000 7fc22fbea700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fc228106c20 con 0x7fc2280f6510 2023-11-17T05:27:45.843 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:45.841+0000 7fc22d986700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fc2280f6510 0x7fc228106730 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:27:45.843 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:45.841+0000 7fc22d986700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fc2280f6510 0x7fc228106730 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.66:48274/0 (socket says 172.21.15.66:48274) 2023-11-17T05:27:45.844 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:45.841+0000 7fc22d986700 1 -- 172.21.15.66:0/3528113476 learned_addr learned my addr 172.21.15.66:0/3528113476 (peer_addr_for_me v2:172.21.15.66:0/0) 2023-11-17T05:27:45.845 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:45.842+0000 7fc22d986700 1 -- 172.21.15.66:0/3528113476 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fc224005040 con 0x7fc2280f6510 2023-11-17T05:27:45.845 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:45.842+0000 7fc22d986700 1 --2- 172.21.15.66:0/3528113476 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fc2280f6510 0x7fc228106730 secure :-1 s=READY pgs=230 cs=0 l=1 rev1=1 rx=0x7fc224013510 tx=0x7fc22400b040).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:27:45.845 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:45.842+0000 7fc21a7fc700 1 -- 172.21.15.66:0/3528113476 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7fc224009320 con 0x7fc2280f6510 2023-11-17T05:27:45.845 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:45.842+0000 7fc22fbea700 1 -- 172.21.15.66:0/3528113476 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fc228106e20 con 0x7fc2280f6510 2023-11-17T05:27:45.845 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:45.842+0000 7fc21a7fc700 1 -- 172.21.15.66:0/3528113476 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(25 keys) v1 ==== 1074+0+0 (secure 0 0 0) 0x7fc224013f00 con 0x7fc2280f6510 2023-11-17T05:27:45.845 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:45.842+0000 7fc21a7fc700 1 -- 172.21.15.66:0/3528113476 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7fc224014d90 con 0x7fc2280f6510 2023-11-17T05:27:45.845 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:45.842+0000 7fc22fbea700 1 -- 172.21.15.66:0/3528113476 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fc2281072b0 con 0x7fc2280f6510 2023-11-17T05:27:45.846 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:45.843+0000 7fc21a7fc700 1 -- 172.21.15.66:0/3528113476 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 17) v1 ==== 36269+0+0 (secure 0 0 0) 0x7fc2240095f0 con 0x7fc2280f6510 2023-11-17T05:27:45.846 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:45.843+0000 7fc22fbea700 1 -- 172.21.15.66:0/3528113476 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fc228045ad0 con 0x7fc2280f6510 2023-11-17T05:27:45.846 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:45.843+0000 7fc21a7fc700 1 --2- 172.21.15.66:0/3528113476 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fc21402ffb0 0x7fc214032470 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:27:45.846 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:45.843+0000 7fc21a7fc700 1 -- 172.21.15.66:0/3528113476 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(5..5 src has 1..5) v4 ==== 1270+0+0 (secure 0 0 0) 0x7fc224041090 con 0x7fc2280f6510 2023-11-17T05:27:45.846 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:45.843+0000 7fc22d185700 1 --2- 172.21.15.66:0/3528113476 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fc21402ffb0 0x7fc214032470 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:27:45.846 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:45.844+0000 7fc22d185700 1 --2- 172.21.15.66:0/3528113476 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fc21402ffb0 0x7fc214032470 secure :-1 s=READY pgs=80 cs=0 l=1 rev1=1 rx=0x7fc21c0096a0 tx=0x7fc21c006b40).ready entity=mgr.14164 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:27:45.850 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:45.849+0000 7fc21a7fc700 1 -- 172.21.15.66:0/3528113476 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+152212 (secure 0 0 0) 0x7fc22406d2b0 con 0x7fc2280f6510 2023-11-17T05:27:46.069 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:46.067+0000 7fc22fbea700 1 -- 172.21.15.66:0/3528113476 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "mon dump", "format": "json"} v 0) v1 -- 0x7fc2280fbf60 con 0x7fc2280f6510 2023-11-17T05:27:46.069 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:46.068+0000 7fc21a7fc700 1 -- 172.21.15.66:0/3528113476 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mon_command_ack([{"prefix": "mon dump", "format": "json"}]=0 dumped monmap epoch 1 v1) v1 ==== 95+0+690 (secure 0 0 0) 0x7fc224014420 con 0x7fc2280f6510 2023-11-17T05:27:46.070 INFO:teuthology.orchestra.run.smithi066.stdout: 2023-11-17T05:27:46.070 INFO:teuthology.orchestra.run.smithi066.stdout:{"epoch":1,"fsid":"ed6470e2-8508-11ee-8dbe-212e2dc638e7","modified":"2023-11-17T05:20:13.934038Z","created":"2023-11-17T05:20:13.934038Z","min_mon_release":16,"min_mon_release_name":"pacific","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging"],"optional":[]},"mons":[{"rank":0,"name":"smithi040","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:3300","nonce":0},{"type":"v1","addr":"172.21.15.40:6789","nonce":0}]},"addr":"172.21.15.40:6789/0","public_addr":"172.21.15.40:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2023-11-17T05:27:46.071 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:46.070+0000 7fc22fbea700 1 -- 172.21.15.66:0/3528113476 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fc21402ffb0 msgr2=0x7fc214032470 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:27:46.071 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:46.070+0000 7fc22fbea700 1 --2- 172.21.15.66:0/3528113476 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fc21402ffb0 0x7fc214032470 secure :-1 s=READY pgs=80 cs=0 l=1 rev1=1 rx=0x7fc21c0096a0 tx=0x7fc21c006b40).stop 2023-11-17T05:27:46.071 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:46.070+0000 7fc22fbea700 1 -- 172.21.15.66:0/3528113476 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fc2280f6510 msgr2=0x7fc228106730 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:27:46.071 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:46.070+0000 7fc22fbea700 1 --2- 172.21.15.66:0/3528113476 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fc2280f6510 0x7fc228106730 secure :-1 s=READY pgs=230 cs=0 l=1 rev1=1 rx=0x7fc224013510 tx=0x7fc22400b040).stop 2023-11-17T05:27:46.071 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:46.070+0000 7fc22fbea700 1 -- 172.21.15.66:0/3528113476 shutdown_connections 2023-11-17T05:27:46.072 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:46.070+0000 7fc22fbea700 1 --2- 172.21.15.66:0/3528113476 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fc21402ffb0 0x7fc214032470 unknown :-1 s=CLOSED pgs=80 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:27:46.072 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:46.070+0000 7fc22fbea700 1 --2- 172.21.15.66:0/3528113476 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fc2280f6510 0x7fc228106730 unknown :-1 s=CLOSED pgs=230 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:27:46.072 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:46.070+0000 7fc22fbea700 1 -- 172.21.15.66:0/3528113476 >> 172.21.15.66:0/3528113476 conn(0x7fc2280f1c70 msgr2=0x7fc2280f2760 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:27:46.072 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:46.070+0000 7fc22fbea700 1 -- 172.21.15.66:0/3528113476 shutdown_connections 2023-11-17T05:27:46.072 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:46.070+0000 7fc22fbea700 1 -- 172.21.15.66:0/3528113476 wait complete. 2023-11-17T05:27:46.072 INFO:teuthology.orchestra.run.smithi066.stderr:dumped monmap epoch 1 2023-11-17T05:27:46.738 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:27:46 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:27:45.220273+0000 mgr.smithi040.ogcprl (mgr.14164) 203 : cluster [DBG] pgmap v191: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:27:46.739 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:27:46 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:27:46.069684+0000 mon.smithi040 (mon.0) 209 : audit [DBG] from='client.? 172.21.15.66:0/3528113476' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-11-17T05:27:47.465 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2023-11-17T05:27:47.465 DEBUG:teuthology.orchestra.run.smithi066:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.4 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ed6470e2-8508-11ee-8dbe-212e2dc638e7 -- ceph mon dump -f json 2023-11-17T05:27:48.605 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:48.602+0000 7f83f3c56700 1 -- 172.21.15.66:0/966501176 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f83ec0f5410 msgr2=0x7f83ec0f5830 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:27:48.605 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:48.602+0000 7f83f3c56700 1 --2- 172.21.15.66:0/966501176 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f83ec0f5410 0x7f83ec0f5830 secure :-1 s=READY pgs=231 cs=0 l=1 rev1=1 rx=0x7f83e8009fb0 tx=0x7f83e8009b30).stop 2023-11-17T05:27:48.605 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:48.602+0000 7f83f3c56700 1 -- 172.21.15.66:0/966501176 shutdown_connections 2023-11-17T05:27:48.605 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:48.602+0000 7f83f3c56700 1 --2- 172.21.15.66:0/966501176 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f83ec0f5410 0x7f83ec0f5830 unknown :-1 s=CLOSED pgs=231 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:27:48.605 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:48.602+0000 7f83f3c56700 1 -- 172.21.15.66:0/966501176 >> 172.21.15.66:0/966501176 conn(0x7f83ec0f0b70 msgr2=0x7f83ec0f2f90 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:27:48.605 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:48.602+0000 7f83f3c56700 1 -- 172.21.15.66:0/966501176 shutdown_connections 2023-11-17T05:27:48.605 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:48.602+0000 7f83f3c56700 1 -- 172.21.15.66:0/966501176 wait complete. 2023-11-17T05:27:48.605 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:48.603+0000 7f83f3c56700 1 Processor -- start 2023-11-17T05:27:48.606 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:48.603+0000 7f83f3c56700 1 -- start start 2023-11-17T05:27:48.606 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:48.603+0000 7f83f3c56700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f83ec0f5410 0x7f83ec105630 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:27:48.606 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:48.603+0000 7f83f3c56700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f83ec105b20 con 0x7f83ec0f5410 2023-11-17T05:27:48.606 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:48.603+0000 7f83f19f2700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f83ec0f5410 0x7f83ec105630 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:27:48.606 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:48.603+0000 7f83f19f2700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f83ec0f5410 0x7f83ec105630 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.66:48296/0 (socket says 172.21.15.66:48296) 2023-11-17T05:27:48.606 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:48.603+0000 7f83f19f2700 1 -- 172.21.15.66:0/1439864867 learned_addr learned my addr 172.21.15.66:0/1439864867 (peer_addr_for_me v2:172.21.15.66:0/0) 2023-11-17T05:27:48.606 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:48.604+0000 7f83f19f2700 1 -- 172.21.15.66:0/1439864867 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f83e8005040 con 0x7f83ec0f5410 2023-11-17T05:27:48.606 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:48.604+0000 7f83f19f2700 1 --2- 172.21.15.66:0/1439864867 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f83ec0f5410 0x7f83ec105630 secure :-1 s=READY pgs=232 cs=0 l=1 rev1=1 rx=0x7f83e8013510 tx=0x7f83e800b040).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:27:48.607 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:48.604+0000 7f83de7fc700 1 -- 172.21.15.66:0/1439864867 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f83e8009320 con 0x7f83ec0f5410 2023-11-17T05:27:48.607 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:48.604+0000 7f83de7fc700 1 -- 172.21.15.66:0/1439864867 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(25 keys) v1 ==== 1074+0+0 (secure 0 0 0) 0x7f83e8013f00 con 0x7f83ec0f5410 2023-11-17T05:27:48.607 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:48.604+0000 7f83f3c56700 1 -- 172.21.15.66:0/1439864867 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f83ec105d20 con 0x7f83ec0f5410 2023-11-17T05:27:48.607 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:48.604+0000 7f83de7fc700 1 -- 172.21.15.66:0/1439864867 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f83e8014d90 con 0x7f83ec0f5410 2023-11-17T05:27:48.608 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:48.604+0000 7f83f3c56700 1 -- 172.21.15.66:0/1439864867 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f83ec1061b0 con 0x7f83ec0f5410 2023-11-17T05:27:48.608 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:48.605+0000 7f83de7fc700 1 -- 172.21.15.66:0/1439864867 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 17) v1 ==== 36269+0+0 (secure 0 0 0) 0x7f83e80095f0 con 0x7f83ec0f5410 2023-11-17T05:27:48.608 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:48.605+0000 7f83f3c56700 1 -- 172.21.15.66:0/1439864867 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f83ec045ad0 con 0x7f83ec0f5410 2023-11-17T05:27:48.608 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:48.605+0000 7f83de7fc700 1 --2- 172.21.15.66:0/1439864867 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f83d802ff60 0x7f83d8032420 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:27:48.608 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:48.605+0000 7f83de7fc700 1 -- 172.21.15.66:0/1439864867 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(5..5 src has 1..5) v4 ==== 1270+0+0 (secure 0 0 0) 0x7f83e80410c0 con 0x7f83ec0f5410 2023-11-17T05:27:48.608 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:48.605+0000 7f83f11f1700 1 --2- 172.21.15.66:0/1439864867 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f83d802ff60 0x7f83d8032420 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:27:48.608 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:48.606+0000 7f83f11f1700 1 --2- 172.21.15.66:0/1439864867 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f83d802ff60 0x7f83d8032420 secure :-1 s=READY pgs=81 cs=0 l=1 rev1=1 rx=0x7f83e00096a0 tx=0x7f83e0006b40).ready entity=mgr.14164 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:27:48.611 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:48.611+0000 7f83de7fc700 1 -- 172.21.15.66:0/1439864867 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+152212 (secure 0 0 0) 0x7f83e8014420 con 0x7f83ec0f5410 2023-11-17T05:27:48.741 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:27:48 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:27:47.220641+0000 mgr.smithi040.ogcprl (mgr.14164) 204 : cluster [DBG] pgmap v192: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:27:48.830 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:48.828+0000 7f83f3c56700 1 -- 172.21.15.66:0/1439864867 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "mon dump", "format": "json"} v 0) v1 -- 0x7f83ec0fae60 con 0x7f83ec0f5410 2023-11-17T05:27:48.830 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:48.829+0000 7f83de7fc700 1 -- 172.21.15.66:0/1439864867 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mon_command_ack([{"prefix": "mon dump", "format": "json"}]=0 dumped monmap epoch 1 v1) v1 ==== 95+0+690 (secure 0 0 0) 0x7f83e801d7f0 con 0x7f83ec0f5410 2023-11-17T05:27:48.830 INFO:teuthology.orchestra.run.smithi066.stdout: 2023-11-17T05:27:48.831 INFO:teuthology.orchestra.run.smithi066.stdout:{"epoch":1,"fsid":"ed6470e2-8508-11ee-8dbe-212e2dc638e7","modified":"2023-11-17T05:20:13.934038Z","created":"2023-11-17T05:20:13.934038Z","min_mon_release":16,"min_mon_release_name":"pacific","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging"],"optional":[]},"mons":[{"rank":0,"name":"smithi040","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:3300","nonce":0},{"type":"v1","addr":"172.21.15.40:6789","nonce":0}]},"addr":"172.21.15.40:6789/0","public_addr":"172.21.15.40:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2023-11-17T05:27:48.832 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:48.831+0000 7f83f3c56700 1 -- 172.21.15.66:0/1439864867 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f83d802ff60 msgr2=0x7f83d8032420 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:27:48.832 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:48.831+0000 7f83f3c56700 1 --2- 172.21.15.66:0/1439864867 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f83d802ff60 0x7f83d8032420 secure :-1 s=READY pgs=81 cs=0 l=1 rev1=1 rx=0x7f83e00096a0 tx=0x7f83e0006b40).stop 2023-11-17T05:27:48.832 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:48.831+0000 7f83f3c56700 1 -- 172.21.15.66:0/1439864867 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f83ec0f5410 msgr2=0x7f83ec105630 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:27:48.832 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:48.831+0000 7f83f3c56700 1 --2- 172.21.15.66:0/1439864867 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f83ec0f5410 0x7f83ec105630 secure :-1 s=READY pgs=232 cs=0 l=1 rev1=1 rx=0x7f83e8013510 tx=0x7f83e800b040).stop 2023-11-17T05:27:48.832 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:48.831+0000 7f83f3c56700 1 -- 172.21.15.66:0/1439864867 shutdown_connections 2023-11-17T05:27:48.833 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:48.831+0000 7f83f3c56700 1 --2- 172.21.15.66:0/1439864867 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f83d802ff60 0x7f83d8032420 unknown :-1 s=CLOSED pgs=81 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:27:48.833 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:48.831+0000 7f83f3c56700 1 --2- 172.21.15.66:0/1439864867 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f83ec0f5410 0x7f83ec105630 unknown :-1 s=CLOSED pgs=232 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:27:48.833 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:48.831+0000 7f83f3c56700 1 -- 172.21.15.66:0/1439864867 >> 172.21.15.66:0/1439864867 conn(0x7f83ec0f0b70 msgr2=0x7f83ec0f1660 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:27:48.833 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:48.831+0000 7f83f3c56700 1 -- 172.21.15.66:0/1439864867 shutdown_connections 2023-11-17T05:27:48.833 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:48.831+0000 7f83f3c56700 1 -- 172.21.15.66:0/1439864867 wait complete. 2023-11-17T05:27:48.833 INFO:teuthology.orchestra.run.smithi066.stderr:dumped monmap epoch 1 2023-11-17T05:27:49.739 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:27:49 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:27:48.830765+0000 mon.smithi040 (mon.0) 210 : audit [DBG] from='client.? 172.21.15.66:0/1439864867' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-11-17T05:27:50.205 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2023-11-17T05:27:50.205 DEBUG:teuthology.orchestra.run.smithi066:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.4 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ed6470e2-8508-11ee-8dbe-212e2dc638e7 -- ceph mon dump -f json 2023-11-17T05:27:50.740 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:27:50 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:27:49.221014+0000 mgr.smithi040.ogcprl (mgr.14164) 205 : cluster [DBG] pgmap v193: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:27:51.406 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:51.404+0000 7fb0e7fff700 1 -- 172.21.15.66:0/3964327904 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fb0e80f51a0 msgr2=0x7fb0e80f55c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:27:51.407 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:51.404+0000 7fb0e7fff700 1 --2- 172.21.15.66:0/3964327904 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fb0e80f51a0 0x7fb0e80f55c0 secure :-1 s=READY pgs=233 cs=0 l=1 rev1=1 rx=0x7fb0d8009fb0 tx=0x7fb0d8009b30).stop 2023-11-17T05:27:51.407 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:51.404+0000 7fb0e7fff700 1 -- 172.21.15.66:0/3964327904 shutdown_connections 2023-11-17T05:27:51.407 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:51.404+0000 7fb0e7fff700 1 --2- 172.21.15.66:0/3964327904 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fb0e80f51a0 0x7fb0e80f55c0 unknown :-1 s=CLOSED pgs=233 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:27:51.407 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:51.404+0000 7fb0e7fff700 1 -- 172.21.15.66:0/3964327904 >> 172.21.15.66:0/3964327904 conn(0x7fb0e80f09a0 msgr2=0x7fb0e80f2dc0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:27:51.407 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:51.404+0000 7fb0e7fff700 1 -- 172.21.15.66:0/3964327904 shutdown_connections 2023-11-17T05:27:51.407 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:51.404+0000 7fb0e7fff700 1 -- 172.21.15.66:0/3964327904 wait complete. 2023-11-17T05:27:51.407 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:51.405+0000 7fb0e7fff700 1 Processor -- start 2023-11-17T05:27:51.407 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:51.405+0000 7fb0e7fff700 1 -- start start 2023-11-17T05:27:51.407 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:51.405+0000 7fb0e7fff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fb0e80f51a0 0x7fb0e81042a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:27:51.408 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:51.405+0000 7fb0e7fff700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fb0e8104790 con 0x7fb0e80f51a0 2023-11-17T05:27:51.408 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:51.405+0000 7fb0e6ffd700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fb0e80f51a0 0x7fb0e81042a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:27:51.408 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:51.405+0000 7fb0e6ffd700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fb0e80f51a0 0x7fb0e81042a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.66:48310/0 (socket says 172.21.15.66:48310) 2023-11-17T05:27:51.408 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:51.405+0000 7fb0e6ffd700 1 -- 172.21.15.66:0/3475802421 learned_addr learned my addr 172.21.15.66:0/3475802421 (peer_addr_for_me v2:172.21.15.66:0/0) 2023-11-17T05:27:51.408 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:51.405+0000 7fb0e6ffd700 1 -- 172.21.15.66:0/3475802421 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fb0d8005040 con 0x7fb0e80f51a0 2023-11-17T05:27:51.408 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:51.405+0000 7fb0e6ffd700 1 --2- 172.21.15.66:0/3475802421 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fb0e80f51a0 0x7fb0e81042a0 secure :-1 s=READY pgs=234 cs=0 l=1 rev1=1 rx=0x7fb0d8013510 tx=0x7fb0d800b040).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:27:51.408 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:51.406+0000 7fb0cf7fe700 1 -- 172.21.15.66:0/3475802421 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7fb0d8009320 con 0x7fb0e80f51a0 2023-11-17T05:27:51.408 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:51.406+0000 7fb0e7fff700 1 -- 172.21.15.66:0/3475802421 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fb0e8104990 con 0x7fb0e80f51a0 2023-11-17T05:27:51.408 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:51.406+0000 7fb0cf7fe700 1 -- 172.21.15.66:0/3475802421 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(25 keys) v1 ==== 1074+0+0 (secure 0 0 0) 0x7fb0d8013f00 con 0x7fb0e80f51a0 2023-11-17T05:27:51.408 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:51.406+0000 7fb0cf7fe700 1 -- 172.21.15.66:0/3475802421 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7fb0d8014dc0 con 0x7fb0e80f51a0 2023-11-17T05:27:51.408 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:51.406+0000 7fb0e7fff700 1 -- 172.21.15.66:0/3475802421 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fb0e8104e20 con 0x7fb0e80f51a0 2023-11-17T05:27:51.409 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:51.407+0000 7fb0cf7fe700 1 -- 172.21.15.66:0/3475802421 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 17) v1 ==== 36269+0+0 (secure 0 0 0) 0x7fb0d8014420 con 0x7fb0e80f51a0 2023-11-17T05:27:51.409 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:51.407+0000 7fb0e7fff700 1 -- 172.21.15.66:0/3475802421 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fb0e8045ad0 con 0x7fb0e80f51a0 2023-11-17T05:27:51.410 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:51.407+0000 7fb0cf7fe700 1 --2- 172.21.15.66:0/3475802421 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fb0d00321b0 0x7fb0d0034670 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:27:51.410 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:51.407+0000 7fb0cf7fe700 1 -- 172.21.15.66:0/3475802421 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(5..5 src has 1..5) v4 ==== 1270+0+0 (secure 0 0 0) 0x7fb0d8041050 con 0x7fb0e80f51a0 2023-11-17T05:27:51.410 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:51.407+0000 7fb0e67fc700 1 --2- 172.21.15.66:0/3475802421 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fb0d00321b0 0x7fb0d0034670 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:27:51.410 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:51.407+0000 7fb0e67fc700 1 --2- 172.21.15.66:0/3475802421 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fb0d00321b0 0x7fb0d0034670 secure :-1 s=READY pgs=82 cs=0 l=1 rev1=1 rx=0x7fb0dc0096a0 tx=0x7fb0dc006b40).ready entity=mgr.14164 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:27:51.411 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:51.411+0000 7fb0cf7fe700 1 -- 172.21.15.66:0/3475802421 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+152212 (secure 0 0 0) 0x7fb0d80095f0 con 0x7fb0e80f51a0 2023-11-17T05:27:51.631 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:51.630+0000 7fb0e7fff700 1 -- 172.21.15.66:0/3475802421 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "mon dump", "format": "json"} v 0) v1 -- 0x7fb0e8105c50 con 0x7fb0e80f51a0 2023-11-17T05:27:51.631 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:51.631+0000 7fb0cf7fe700 1 -- 172.21.15.66:0/3475802421 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mon_command_ack([{"prefix": "mon dump", "format": "json"}]=0 dumped monmap epoch 1 v1) v1 ==== 95+0+690 (secure 0 0 0) 0x7fb0d801db70 con 0x7fb0e80f51a0 2023-11-17T05:27:51.632 INFO:teuthology.orchestra.run.smithi066.stdout: 2023-11-17T05:27:51.632 INFO:teuthology.orchestra.run.smithi066.stdout:{"epoch":1,"fsid":"ed6470e2-8508-11ee-8dbe-212e2dc638e7","modified":"2023-11-17T05:20:13.934038Z","created":"2023-11-17T05:20:13.934038Z","min_mon_release":16,"min_mon_release_name":"pacific","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging"],"optional":[]},"mons":[{"rank":0,"name":"smithi040","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:3300","nonce":0},{"type":"v1","addr":"172.21.15.40:6789","nonce":0}]},"addr":"172.21.15.40:6789/0","public_addr":"172.21.15.40:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2023-11-17T05:27:51.633 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:51.633+0000 7fb0e7fff700 1 -- 172.21.15.66:0/3475802421 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fb0d00321b0 msgr2=0x7fb0d0034670 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:27:51.633 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:51.633+0000 7fb0e7fff700 1 --2- 172.21.15.66:0/3475802421 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fb0d00321b0 0x7fb0d0034670 secure :-1 s=READY pgs=82 cs=0 l=1 rev1=1 rx=0x7fb0dc0096a0 tx=0x7fb0dc006b40).stop 2023-11-17T05:27:51.633 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:51.633+0000 7fb0e7fff700 1 -- 172.21.15.66:0/3475802421 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fb0e80f51a0 msgr2=0x7fb0e81042a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:27:51.633 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:51.633+0000 7fb0e7fff700 1 --2- 172.21.15.66:0/3475802421 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fb0e80f51a0 0x7fb0e81042a0 secure :-1 s=READY pgs=234 cs=0 l=1 rev1=1 rx=0x7fb0d8013510 tx=0x7fb0d800b040).stop 2023-11-17T05:27:51.633 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:51.633+0000 7fb0e7fff700 1 -- 172.21.15.66:0/3475802421 shutdown_connections 2023-11-17T05:27:51.633 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:51.633+0000 7fb0e7fff700 1 --2- 172.21.15.66:0/3475802421 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fb0d00321b0 0x7fb0d0034670 unknown :-1 s=CLOSED pgs=82 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:27:51.633 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:51.633+0000 7fb0e7fff700 1 --2- 172.21.15.66:0/3475802421 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fb0e80f51a0 0x7fb0e81042a0 unknown :-1 s=CLOSED pgs=234 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:27:51.634 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:51.633+0000 7fb0e7fff700 1 -- 172.21.15.66:0/3475802421 >> 172.21.15.66:0/3475802421 conn(0x7fb0e80f09a0 msgr2=0x7fb0e80f1400 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:27:51.634 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:51.633+0000 7fb0e7fff700 1 -- 172.21.15.66:0/3475802421 shutdown_connections 2023-11-17T05:27:51.634 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:51.633+0000 7fb0e7fff700 1 -- 172.21.15.66:0/3475802421 wait complete. 2023-11-17T05:27:51.634 INFO:teuthology.orchestra.run.smithi066.stderr:dumped monmap epoch 1 2023-11-17T05:27:52.742 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:27:52 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:27:51.221449+0000 mgr.smithi040.ogcprl (mgr.14164) 206 : cluster [DBG] pgmap v194: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:27:52.743 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:27:52 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:27:51.631826+0000 mon.smithi040 (mon.0) 211 : audit [DBG] from='client.? 172.21.15.66:0/3475802421' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-11-17T05:27:53.037 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2023-11-17T05:27:53.037 DEBUG:teuthology.orchestra.run.smithi066:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.4 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ed6470e2-8508-11ee-8dbe-212e2dc638e7 -- ceph mon dump -f json 2023-11-17T05:27:54.221 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:54.220+0000 7fe1eb0f1700 1 -- 172.21.15.66:0/609676761 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fe1e40f5410 msgr2=0x7fe1e40f5830 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:27:54.221 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:54.220+0000 7fe1eb0f1700 1 --2- 172.21.15.66:0/609676761 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fe1e40f5410 0x7fe1e40f5830 secure :-1 s=READY pgs=235 cs=0 l=1 rev1=1 rx=0x7fe1d8009fb0 tx=0x7fe1d8009b30).stop 2023-11-17T05:27:54.221 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:54.220+0000 7fe1eb0f1700 1 -- 172.21.15.66:0/609676761 shutdown_connections 2023-11-17T05:27:54.222 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:54.220+0000 7fe1eb0f1700 1 --2- 172.21.15.66:0/609676761 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fe1e40f5410 0x7fe1e40f5830 unknown :-1 s=CLOSED pgs=235 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:27:54.222 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:54.220+0000 7fe1eb0f1700 1 -- 172.21.15.66:0/609676761 >> 172.21.15.66:0/609676761 conn(0x7fe1e40f0b70 msgr2=0x7fe1e40f2f90 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:27:54.222 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:54.220+0000 7fe1eb0f1700 1 -- 172.21.15.66:0/609676761 shutdown_connections 2023-11-17T05:27:54.222 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:54.220+0000 7fe1eb0f1700 1 -- 172.21.15.66:0/609676761 wait complete. 2023-11-17T05:27:54.222 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:54.221+0000 7fe1eb0f1700 1 Processor -- start 2023-11-17T05:27:54.222 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:54.221+0000 7fe1eb0f1700 1 -- start start 2023-11-17T05:27:54.222 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:54.221+0000 7fe1eb0f1700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fe1e40f5410 0x7fe1e4106720 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:27:54.222 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:54.221+0000 7fe1eb0f1700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fe1e4106c10 con 0x7fe1e40f5410 2023-11-17T05:27:54.224 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:54.222+0000 7fe1e8e8d700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fe1e40f5410 0x7fe1e4106720 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:27:54.224 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:54.222+0000 7fe1e8e8d700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fe1e40f5410 0x7fe1e4106720 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.66:40352/0 (socket says 172.21.15.66:40352) 2023-11-17T05:27:54.224 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:54.222+0000 7fe1e8e8d700 1 -- 172.21.15.66:0/3450651635 learned_addr learned my addr 172.21.15.66:0/3450651635 (peer_addr_for_me v2:172.21.15.66:0/0) 2023-11-17T05:27:54.224 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:54.222+0000 7fe1e8e8d700 1 -- 172.21.15.66:0/3450651635 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fe1d8005040 con 0x7fe1e40f5410 2023-11-17T05:27:54.224 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:54.222+0000 7fe1e8e8d700 1 --2- 172.21.15.66:0/3450651635 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fe1e40f5410 0x7fe1e4106720 secure :-1 s=READY pgs=236 cs=0 l=1 rev1=1 rx=0x7fe1d8013510 tx=0x7fe1d800b040).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:27:54.224 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:54.222+0000 7fe1e17fa700 1 -- 172.21.15.66:0/3450651635 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7fe1d8009320 con 0x7fe1e40f5410 2023-11-17T05:27:54.224 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:54.222+0000 7fe1eb0f1700 1 -- 172.21.15.66:0/3450651635 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fe1e4106e10 con 0x7fe1e40f5410 2023-11-17T05:27:54.224 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:54.222+0000 7fe1e17fa700 1 -- 172.21.15.66:0/3450651635 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(25 keys) v1 ==== 1074+0+0 (secure 0 0 0) 0x7fe1d8013f00 con 0x7fe1e40f5410 2023-11-17T05:27:54.224 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:54.222+0000 7fe1e17fa700 1 -- 172.21.15.66:0/3450651635 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7fe1d8014d90 con 0x7fe1e40f5410 2023-11-17T05:27:54.224 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:54.222+0000 7fe1eb0f1700 1 -- 172.21.15.66:0/3450651635 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fe1e41072a0 con 0x7fe1e40f5410 2023-11-17T05:27:54.224 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:54.223+0000 7fe1e17fa700 1 -- 172.21.15.66:0/3450651635 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 17) v1 ==== 36269+0+0 (secure 0 0 0) 0x7fe1d8014420 con 0x7fe1e40f5410 2023-11-17T05:27:54.225 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:54.223+0000 7fe1eb0f1700 1 -- 172.21.15.66:0/3450651635 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fe1e4045ad0 con 0x7fe1e40f5410 2023-11-17T05:27:54.225 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:54.223+0000 7fe1e17fa700 1 --2- 172.21.15.66:0/3450651635 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fe1d402ff60 0x7fe1d4032420 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:27:54.225 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:54.223+0000 7fe1e17fa700 1 -- 172.21.15.66:0/3450651635 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(5..5 src has 1..5) v4 ==== 1270+0+0 (secure 0 0 0) 0x7fe1d8040ff0 con 0x7fe1e40f5410 2023-11-17T05:27:54.225 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:54.224+0000 7fe1e3fff700 1 --2- 172.21.15.66:0/3450651635 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fe1d402ff60 0x7fe1d4032420 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:27:54.225 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:54.224+0000 7fe1e3fff700 1 --2- 172.21.15.66:0/3450651635 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fe1d402ff60 0x7fe1d4032420 secure :-1 s=READY pgs=83 cs=0 l=1 rev1=1 rx=0x7fe1d00096a0 tx=0x7fe1d0006b40).ready entity=mgr.14164 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:27:54.228 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:54.228+0000 7fe1e17fa700 1 -- 172.21.15.66:0/3450651635 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+152212 (secure 0 0 0) 0x7fe1d80095f0 con 0x7fe1e40f5410 2023-11-17T05:27:54.449 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:54.449+0000 7fe1eb0f1700 1 -- 172.21.15.66:0/3450651635 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "mon dump", "format": "json"} v 0) v1 -- 0x7fe1e4108150 con 0x7fe1e40f5410 2023-11-17T05:27:54.450 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:54.449+0000 7fe1e17fa700 1 -- 172.21.15.66:0/3450651635 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mon_command_ack([{"prefix": "mon dump", "format": "json"}]=0 dumped monmap epoch 1 v1) v1 ==== 95+0+690 (secure 0 0 0) 0x7fe1d801db70 con 0x7fe1e40f5410 2023-11-17T05:27:54.450 INFO:teuthology.orchestra.run.smithi066.stdout: 2023-11-17T05:27:54.450 INFO:teuthology.orchestra.run.smithi066.stdout:{"epoch":1,"fsid":"ed6470e2-8508-11ee-8dbe-212e2dc638e7","modified":"2023-11-17T05:20:13.934038Z","created":"2023-11-17T05:20:13.934038Z","min_mon_release":16,"min_mon_release_name":"pacific","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging"],"optional":[]},"mons":[{"rank":0,"name":"smithi040","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:3300","nonce":0},{"type":"v1","addr":"172.21.15.40:6789","nonce":0}]},"addr":"172.21.15.40:6789/0","public_addr":"172.21.15.40:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2023-11-17T05:27:54.452 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:54.451+0000 7fe1eb0f1700 1 -- 172.21.15.66:0/3450651635 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fe1d402ff60 msgr2=0x7fe1d4032420 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:27:54.499 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:54.451+0000 7fe1eb0f1700 1 --2- 172.21.15.66:0/3450651635 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fe1d402ff60 0x7fe1d4032420 secure :-1 s=READY pgs=83 cs=0 l=1 rev1=1 rx=0x7fe1d00096a0 tx=0x7fe1d0006b40).stop 2023-11-17T05:27:54.499 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:54.451+0000 7fe1eb0f1700 1 -- 172.21.15.66:0/3450651635 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fe1e40f5410 msgr2=0x7fe1e4106720 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:27:54.499 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:54.451+0000 7fe1eb0f1700 1 --2- 172.21.15.66:0/3450651635 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fe1e40f5410 0x7fe1e4106720 secure :-1 s=READY pgs=236 cs=0 l=1 rev1=1 rx=0x7fe1d8013510 tx=0x7fe1d800b040).stop 2023-11-17T05:27:54.500 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:54.451+0000 7fe1eb0f1700 1 -- 172.21.15.66:0/3450651635 shutdown_connections 2023-11-17T05:27:54.500 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:54.451+0000 7fe1eb0f1700 1 --2- 172.21.15.66:0/3450651635 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fe1d402ff60 0x7fe1d4032420 unknown :-1 s=CLOSED pgs=83 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:27:54.500 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:54.451+0000 7fe1eb0f1700 1 --2- 172.21.15.66:0/3450651635 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fe1e40f5410 0x7fe1e4106720 unknown :-1 s=CLOSED pgs=236 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:27:54.500 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:54.451+0000 7fe1eb0f1700 1 -- 172.21.15.66:0/3450651635 >> 172.21.15.66:0/3450651635 conn(0x7fe1e40f0b70 msgr2=0x7fe1e405fe60 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:27:54.500 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:54.451+0000 7fe1eb0f1700 1 -- 172.21.15.66:0/3450651635 shutdown_connections 2023-11-17T05:27:54.500 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:54.451+0000 7fe1eb0f1700 1 -- 172.21.15.66:0/3450651635 wait complete. 2023-11-17T05:27:54.500 INFO:teuthology.orchestra.run.smithi066.stderr:dumped monmap epoch 1 2023-11-17T05:27:54.739 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:27:54 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:27:53.221814+0000 mgr.smithi040.ogcprl (mgr.14164) 207 : cluster [DBG] pgmap v195: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:27:54.739 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:27:54 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:27:54.450011+0000 mon.smithi040 (mon.0) 212 : audit [DBG] from='client.? 172.21.15.66:0/3450651635' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-11-17T05:27:55.831 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2023-11-17T05:27:55.831 DEBUG:teuthology.orchestra.run.smithi066:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.4 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ed6470e2-8508-11ee-8dbe-212e2dc638e7 -- ceph mon dump -f json 2023-11-17T05:27:56.739 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:27:56 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:27:55.222191+0000 mgr.smithi040.ogcprl (mgr.14164) 208 : cluster [DBG] pgmap v196: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:27:57.016 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:57.015+0000 7f4652431700 1 -- 172.21.15.66:0/3293436309 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f464c0f78e0 msgr2=0x7f464c0f7d00 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:27:57.016 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:57.015+0000 7f4652431700 1 --2- 172.21.15.66:0/3293436309 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f464c0f78e0 0x7f464c0f7d00 secure :-1 s=READY pgs=237 cs=0 l=1 rev1=1 rx=0x7f4648009fb0 tx=0x7f4648009b30).stop 2023-11-17T05:27:57.016 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:57.015+0000 7f4652431700 1 -- 172.21.15.66:0/3293436309 shutdown_connections 2023-11-17T05:27:57.016 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:57.015+0000 7f4652431700 1 --2- 172.21.15.66:0/3293436309 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f464c0f78e0 0x7f464c0f7d00 unknown :-1 s=CLOSED pgs=237 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:27:57.016 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:57.015+0000 7f4652431700 1 -- 172.21.15.66:0/3293436309 >> 172.21.15.66:0/3293436309 conn(0x7f464c0f0120 msgr2=0x7f464c0f2540 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:27:57.016 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:57.016+0000 7f4652431700 1 -- 172.21.15.66:0/3293436309 shutdown_connections 2023-11-17T05:27:57.016 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:57.016+0000 7f4652431700 1 -- 172.21.15.66:0/3293436309 wait complete. 2023-11-17T05:27:57.018 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:57.016+0000 7f4652431700 1 Processor -- start 2023-11-17T05:27:57.018 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:57.016+0000 7f4652431700 1 -- start start 2023-11-17T05:27:57.018 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:57.016+0000 7f4652431700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f464c0f78e0 0x7f464c1064e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:27:57.018 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:57.016+0000 7f4652431700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f464c1069d0 con 0x7f464c0f78e0 2023-11-17T05:27:57.018 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:57.017+0000 7f465142f700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f464c0f78e0 0x7f464c1064e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:27:57.018 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:57.017+0000 7f465142f700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f464c0f78e0 0x7f464c1064e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.66:40370/0 (socket says 172.21.15.66:40370) 2023-11-17T05:27:57.019 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:57.017+0000 7f465142f700 1 -- 172.21.15.66:0/2842199224 learned_addr learned my addr 172.21.15.66:0/2842199224 (peer_addr_for_me v2:172.21.15.66:0/0) 2023-11-17T05:27:57.019 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:57.017+0000 7f465142f700 1 -- 172.21.15.66:0/2842199224 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f4648005040 con 0x7f464c0f78e0 2023-11-17T05:27:57.019 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:57.017+0000 7f465142f700 1 --2- 172.21.15.66:0/2842199224 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f464c0f78e0 0x7f464c1064e0 secure :-1 s=READY pgs=238 cs=0 l=1 rev1=1 rx=0x7f4648013510 tx=0x7f464800b040).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:27:57.019 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:57.017+0000 7f4641ffb700 1 -- 172.21.15.66:0/2842199224 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f4648009320 con 0x7f464c0f78e0 2023-11-17T05:27:57.019 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:57.017+0000 7f4652431700 1 -- 172.21.15.66:0/2842199224 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f464c106bd0 con 0x7f464c0f78e0 2023-11-17T05:27:57.019 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:57.017+0000 7f4641ffb700 1 -- 172.21.15.66:0/2842199224 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(25 keys) v1 ==== 1074+0+0 (secure 0 0 0) 0x7f4648013f00 con 0x7f464c0f78e0 2023-11-17T05:27:57.019 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:57.017+0000 7f4641ffb700 1 -- 172.21.15.66:0/2842199224 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f4648014dc0 con 0x7f464c0f78e0 2023-11-17T05:27:57.019 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:57.017+0000 7f4652431700 1 -- 172.21.15.66:0/2842199224 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f464c107060 con 0x7f464c0f78e0 2023-11-17T05:27:57.020 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:57.018+0000 7f4641ffb700 1 -- 172.21.15.66:0/2842199224 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 17) v1 ==== 36269+0+0 (secure 0 0 0) 0x7f464801d3d0 con 0x7f464c0f78e0 2023-11-17T05:27:57.020 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:57.018+0000 7f4652431700 1 -- 172.21.15.66:0/2842199224 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f464c045ad0 con 0x7f464c0f78e0 2023-11-17T05:27:57.020 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:57.018+0000 7f4641ffb700 1 --2- 172.21.15.66:0/2842199224 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f463802ff60 0x7f4638032420 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:27:57.020 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:57.019+0000 7f4641ffb700 1 -- 172.21.15.66:0/2842199224 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(5..5 src has 1..5) v4 ==== 1270+0+0 (secure 0 0 0) 0x7f4648041340 con 0x7f464c0f78e0 2023-11-17T05:27:57.021 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:57.019+0000 7f4650c2e700 1 --2- 172.21.15.66:0/2842199224 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f463802ff60 0x7f4638032420 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:27:57.021 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:57.019+0000 7f4650c2e700 1 --2- 172.21.15.66:0/2842199224 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f463802ff60 0x7f4638032420 secure :-1 s=READY pgs=84 cs=0 l=1 rev1=1 rx=0x7f463c0096a0 tx=0x7f463c006b40).ready entity=mgr.14164 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:27:57.023 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:57.023+0000 7f4641ffb700 1 -- 172.21.15.66:0/2842199224 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+152212 (secure 0 0 0) 0x7f464801cb10 con 0x7f464c0f78e0 2023-11-17T05:27:57.242 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:57.241+0000 7f4652431700 1 -- 172.21.15.66:0/2842199224 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "mon dump", "format": "json"} v 0) v1 -- 0x7f464c107eb0 con 0x7f464c0f78e0 2023-11-17T05:27:57.242 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:57.242+0000 7f4641ffb700 1 -- 172.21.15.66:0/2842199224 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mon_command_ack([{"prefix": "mon dump", "format": "json"}]=0 dumped monmap epoch 1 v1) v1 ==== 95+0+690 (secure 0 0 0) 0x7f464801cb10 con 0x7f464c0f78e0 2023-11-17T05:27:57.242 INFO:teuthology.orchestra.run.smithi066.stdout: 2023-11-17T05:27:57.243 INFO:teuthology.orchestra.run.smithi066.stdout:{"epoch":1,"fsid":"ed6470e2-8508-11ee-8dbe-212e2dc638e7","modified":"2023-11-17T05:20:13.934038Z","created":"2023-11-17T05:20:13.934038Z","min_mon_release":16,"min_mon_release_name":"pacific","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging"],"optional":[]},"mons":[{"rank":0,"name":"smithi040","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:3300","nonce":0},{"type":"v1","addr":"172.21.15.40:6789","nonce":0}]},"addr":"172.21.15.40:6789/0","public_addr":"172.21.15.40:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2023-11-17T05:27:57.244 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:57.243+0000 7f4652431700 1 -- 172.21.15.66:0/2842199224 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f463802ff60 msgr2=0x7f4638032420 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:27:57.244 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:57.243+0000 7f4652431700 1 --2- 172.21.15.66:0/2842199224 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f463802ff60 0x7f4638032420 secure :-1 s=READY pgs=84 cs=0 l=1 rev1=1 rx=0x7f463c0096a0 tx=0x7f463c006b40).stop 2023-11-17T05:27:57.244 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:57.243+0000 7f4652431700 1 -- 172.21.15.66:0/2842199224 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f464c0f78e0 msgr2=0x7f464c1064e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:27:57.244 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:57.243+0000 7f4652431700 1 --2- 172.21.15.66:0/2842199224 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f464c0f78e0 0x7f464c1064e0 secure :-1 s=READY pgs=238 cs=0 l=1 rev1=1 rx=0x7f4648013510 tx=0x7f464800b040).stop 2023-11-17T05:27:57.244 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:57.243+0000 7f4652431700 1 -- 172.21.15.66:0/2842199224 shutdown_connections 2023-11-17T05:27:57.244 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:57.243+0000 7f4652431700 1 --2- 172.21.15.66:0/2842199224 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f463802ff60 0x7f4638032420 unknown :-1 s=CLOSED pgs=84 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:27:57.244 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:57.243+0000 7f4652431700 1 --2- 172.21.15.66:0/2842199224 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f464c0f78e0 0x7f464c1064e0 unknown :-1 s=CLOSED pgs=238 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:27:57.245 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:57.243+0000 7f4652431700 1 -- 172.21.15.66:0/2842199224 >> 172.21.15.66:0/2842199224 conn(0x7f464c0f0120 msgr2=0x7f464c0f0ba0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:27:57.245 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:57.244+0000 7f4652431700 1 -- 172.21.15.66:0/2842199224 shutdown_connections 2023-11-17T05:27:57.245 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:57.244+0000 7f4652431700 1 -- 172.21.15.66:0/2842199224 wait complete. 2023-11-17T05:27:57.245 INFO:teuthology.orchestra.run.smithi066.stderr:dumped monmap epoch 1 2023-11-17T05:27:57.739 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:27:57 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:27:57.242568+0000 mon.smithi040 (mon.0) 213 : audit [DBG] from='client.? 172.21.15.66:0/2842199224' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-11-17T05:27:58.639 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2023-11-17T05:27:58.639 DEBUG:teuthology.orchestra.run.smithi066:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.4 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ed6470e2-8508-11ee-8dbe-212e2dc638e7 -- ceph mon dump -f json 2023-11-17T05:27:58.739 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:27:58 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:27:57.222623+0000 mgr.smithi040.ogcprl (mgr.14164) 209 : cluster [DBG] pgmap v197: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:27:59.840 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:59.839+0000 7fc1a60bf700 1 -- 172.21.15.66:0/3201805590 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fc1a0060280 msgr2=0x7fc1a00f90f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:27:59.840 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:59.839+0000 7fc1a60bf700 1 --2- 172.21.15.66:0/3201805590 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fc1a0060280 0x7fc1a00f90f0 secure :-1 s=READY pgs=239 cs=0 l=1 rev1=1 rx=0x7fc190009fb0 tx=0x7fc190009b30).stop 2023-11-17T05:27:59.840 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:59.839+0000 7fc1a60bf700 1 -- 172.21.15.66:0/3201805590 shutdown_connections 2023-11-17T05:27:59.875 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:59.839+0000 7fc1a60bf700 1 --2- 172.21.15.66:0/3201805590 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fc1a0060280 0x7fc1a00f90f0 unknown :-1 s=CLOSED pgs=239 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:27:59.875 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:59.839+0000 7fc1a60bf700 1 -- 172.21.15.66:0/3201805590 >> 172.21.15.66:0/3201805590 conn(0x7fc1a00f0b70 msgr2=0x7fc1a00f2f90 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:27:59.875 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:59.839+0000 7fc1a60bf700 1 -- 172.21.15.66:0/3201805590 shutdown_connections 2023-11-17T05:27:59.875 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:59.839+0000 7fc1a60bf700 1 -- 172.21.15.66:0/3201805590 wait complete. 2023-11-17T05:27:59.875 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:59.840+0000 7fc1a60bf700 1 Processor -- start 2023-11-17T05:27:59.875 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:59.840+0000 7fc1a60bf700 1 -- start start 2023-11-17T05:27:59.877 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:59.840+0000 7fc1a60bf700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fc1a0060280 0x7fc1a01066e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:27:59.946 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:59.840+0000 7fc1a60bf700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fc1a0106bd0 con 0x7fc1a0060280 2023-11-17T05:28:00.273 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:59.840+0000 7fc19f7fe700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fc1a0060280 0x7fc1a01066e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:28:00.273 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:59.840+0000 7fc19f7fe700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fc1a0060280 0x7fc1a01066e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.66:40388/0 (socket says 172.21.15.66:40388) 2023-11-17T05:28:00.274 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:59.840+0000 7fc19f7fe700 1 -- 172.21.15.66:0/4137337313 learned_addr learned my addr 172.21.15.66:0/4137337313 (peer_addr_for_me v2:172.21.15.66:0/0) 2023-11-17T05:28:00.274 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:59.841+0000 7fc19f7fe700 1 -- 172.21.15.66:0/4137337313 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fc190005040 con 0x7fc1a0060280 2023-11-17T05:28:00.274 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:59.841+0000 7fc19f7fe700 1 --2- 172.21.15.66:0/4137337313 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fc1a0060280 0x7fc1a01066e0 secure :-1 s=READY pgs=240 cs=0 l=1 rev1=1 rx=0x7fc190013510 tx=0x7fc19000b040).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:28:00.274 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:59.841+0000 7fc197fff700 1 -- 172.21.15.66:0/4137337313 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7fc190009320 con 0x7fc1a0060280 2023-11-17T05:28:00.274 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:59.841+0000 7fc197fff700 1 -- 172.21.15.66:0/4137337313 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(25 keys) v1 ==== 1074+0+0 (secure 0 0 0) 0x7fc190013f00 con 0x7fc1a0060280 2023-11-17T05:28:00.274 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:59.841+0000 7fc1a60bf700 1 -- 172.21.15.66:0/4137337313 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fc1a0106dd0 con 0x7fc1a0060280 2023-11-17T05:28:00.274 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:59.841+0000 7fc197fff700 1 -- 172.21.15.66:0/4137337313 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7fc190014dc0 con 0x7fc1a0060280 2023-11-17T05:28:00.274 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:59.841+0000 7fc1a60bf700 1 -- 172.21.15.66:0/4137337313 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fc1a0107260 con 0x7fc1a0060280 2023-11-17T05:28:00.274 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:59.842+0000 7fc197fff700 1 -- 172.21.15.66:0/4137337313 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 17) v1 ==== 36269+0+0 (secure 0 0 0) 0x7fc1900095f0 con 0x7fc1a0060280 2023-11-17T05:28:00.274 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:59.842+0000 7fc197fff700 1 --2- 172.21.15.66:0/4137337313 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fc18c02ffb0 0x7fc18c032470 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:28:00.274 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:59.842+0000 7fc1a60bf700 1 -- 172.21.15.66:0/4137337313 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fc1a0045ad0 con 0x7fc1a0060280 2023-11-17T05:28:00.274 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:59.842+0000 7fc197fff700 1 -- 172.21.15.66:0/4137337313 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(5..5 src has 1..5) v4 ==== 1270+0+0 (secure 0 0 0) 0x7fc190041070 con 0x7fc1a0060280 2023-11-17T05:28:00.275 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:59.842+0000 7fc19effd700 1 --2- 172.21.15.66:0/4137337313 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fc18c02ffb0 0x7fc18c032470 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:28:00.275 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:59.843+0000 7fc19effd700 1 --2- 172.21.15.66:0/4137337313 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fc18c02ffb0 0x7fc18c032470 secure :-1 s=READY pgs=85 cs=0 l=1 rev1=1 rx=0x7fc1880096a0 tx=0x7fc188006b40).ready entity=mgr.14164 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:28:00.275 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:27:59.847+0000 7fc197fff700 1 -- 172.21.15.66:0/4137337313 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+152212 (secure 0 0 0) 0x7fc19001d7f0 con 0x7fc1a0060280 2023-11-17T05:28:00.280 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:00.068+0000 7fc1a60bf700 1 -- 172.21.15.66:0/4137337313 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "mon dump", "format": "json"} v 0) v1 -- 0x7fc1a0108110 con 0x7fc1a0060280 2023-11-17T05:28:00.677 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:00.068+0000 7fc197fff700 1 -- 172.21.15.66:0/4137337313 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mon_command_ack([{"prefix": "mon dump", "format": "json"}]=0 dumped monmap epoch 1 v1) v1 ==== 95+0+690 (secure 0 0 0) 0x7fc19002b020 con 0x7fc1a0060280 2023-11-17T05:28:00.677 INFO:teuthology.orchestra.run.smithi066.stdout: 2023-11-17T05:28:00.677 INFO:teuthology.orchestra.run.smithi066.stdout:{"epoch":1,"fsid":"ed6470e2-8508-11ee-8dbe-212e2dc638e7","modified":"2023-11-17T05:20:13.934038Z","created":"2023-11-17T05:20:13.934038Z","min_mon_release":16,"min_mon_release_name":"pacific","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging"],"optional":[]},"mons":[{"rank":0,"name":"smithi040","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:3300","nonce":0},{"type":"v1","addr":"172.21.15.40:6789","nonce":0}]},"addr":"172.21.15.40:6789/0","public_addr":"172.21.15.40:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2023-11-17T05:28:00.682 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:00.070+0000 7fc1a60bf700 1 -- 172.21.15.66:0/4137337313 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fc18c02ffb0 msgr2=0x7fc18c032470 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:28:00.723 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:00.070+0000 7fc1a60bf700 1 --2- 172.21.15.66:0/4137337313 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fc18c02ffb0 0x7fc18c032470 secure :-1 s=READY pgs=85 cs=0 l=1 rev1=1 rx=0x7fc1880096a0 tx=0x7fc188006b40).stop 2023-11-17T05:28:00.723 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:00.070+0000 7fc1a60bf700 1 -- 172.21.15.66:0/4137337313 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fc1a0060280 msgr2=0x7fc1a01066e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:28:00.723 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:00.070+0000 7fc1a60bf700 1 --2- 172.21.15.66:0/4137337313 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fc1a0060280 0x7fc1a01066e0 secure :-1 s=READY pgs=240 cs=0 l=1 rev1=1 rx=0x7fc190013510 tx=0x7fc19000b040).stop 2023-11-17T05:28:00.723 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:00.070+0000 7fc1a60bf700 1 -- 172.21.15.66:0/4137337313 shutdown_connections 2023-11-17T05:28:00.723 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:00.070+0000 7fc1a60bf700 1 --2- 172.21.15.66:0/4137337313 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fc18c02ffb0 0x7fc18c032470 unknown :-1 s=CLOSED pgs=85 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:28:00.723 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:00.070+0000 7fc1a60bf700 1 --2- 172.21.15.66:0/4137337313 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fc1a0060280 0x7fc1a01066e0 unknown :-1 s=CLOSED pgs=240 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:28:00.723 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:00.070+0000 7fc1a60bf700 1 -- 172.21.15.66:0/4137337313 >> 172.21.15.66:0/4137337313 conn(0x7fc1a00f0b70 msgr2=0x7fc1a00f1660 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:28:00.723 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:00.070+0000 7fc1a60bf700 1 -- 172.21.15.66:0/4137337313 shutdown_connections 2023-11-17T05:28:00.724 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:00.070+0000 7fc1a60bf700 1 -- 172.21.15.66:0/4137337313 wait complete. 2023-11-17T05:28:00.724 INFO:teuthology.orchestra.run.smithi066.stderr:dumped monmap epoch 1 2023-11-17T05:28:00.739 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:28:00 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:27:59.223057+0000 mgr.smithi040.ogcprl (mgr.14164) 210 : cluster [DBG] pgmap v198: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:28:00.739 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:28:00 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:28:00.069187+0000 mon.smithi040 (mon.0) 214 : audit [DBG] from='client.? 172.21.15.66:0/4137337313' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-11-17T05:28:01.725 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2023-11-17T05:28:01.807 DEBUG:teuthology.orchestra.run.smithi066:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.4 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ed6470e2-8508-11ee-8dbe-212e2dc638e7 -- ceph mon dump -f json 2023-11-17T05:28:02.738 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:28:02 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:28:01.223381+0000 mgr.smithi040.ogcprl (mgr.14164) 211 : cluster [DBG] pgmap v199: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:28:02.978 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:02.978+0000 7f833c336700 1 -- 172.21.15.66:0/537777644 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f83340f5410 msgr2=0x7f83340f5830 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:28:02.978 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:02.978+0000 7f833c336700 1 --2- 172.21.15.66:0/537777644 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f83340f5410 0x7f83340f5830 secure :-1 s=READY pgs=241 cs=0 l=1 rev1=1 rx=0x7f8330009fb0 tx=0x7f8330009b30).stop 2023-11-17T05:28:02.978 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:02.978+0000 7f833c336700 1 -- 172.21.15.66:0/537777644 shutdown_connections 2023-11-17T05:28:02.978 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:02.978+0000 7f833c336700 1 --2- 172.21.15.66:0/537777644 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f83340f5410 0x7f83340f5830 unknown :-1 s=CLOSED pgs=241 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:28:02.978 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:02.978+0000 7f833c336700 1 -- 172.21.15.66:0/537777644 >> 172.21.15.66:0/537777644 conn(0x7f83340f0b70 msgr2=0x7f83340f2f90 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:28:02.979 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:02.978+0000 7f833c336700 1 -- 172.21.15.66:0/537777644 shutdown_connections 2023-11-17T05:28:02.979 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:02.978+0000 7f833c336700 1 -- 172.21.15.66:0/537777644 wait complete. 2023-11-17T05:28:02.980 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:02.978+0000 7f833c336700 1 Processor -- start 2023-11-17T05:28:02.980 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:02.978+0000 7f833c336700 1 -- start start 2023-11-17T05:28:02.981 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:02.979+0000 7f833c336700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f83340f5410 0x7f8334109430 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:28:02.981 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:02.979+0000 7f833c336700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f8334060240 con 0x7f83340f5410 2023-11-17T05:28:02.981 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:02.979+0000 7f833a0d2700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f83340f5410 0x7f8334109430 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:28:02.981 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:02.979+0000 7f833a0d2700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f83340f5410 0x7f8334109430 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.66:50506/0 (socket says 172.21.15.66:50506) 2023-11-17T05:28:02.981 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:02.979+0000 7f833a0d2700 1 -- 172.21.15.66:0/972155620 learned_addr learned my addr 172.21.15.66:0/972155620 (peer_addr_for_me v2:172.21.15.66:0/0) 2023-11-17T05:28:02.981 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:02.979+0000 7f833a0d2700 1 -- 172.21.15.66:0/972155620 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f8330005040 con 0x7f83340f5410 2023-11-17T05:28:02.981 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:02.979+0000 7f833a0d2700 1 --2- 172.21.15.66:0/972155620 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f83340f5410 0x7f8334109430 secure :-1 s=READY pgs=242 cs=0 l=1 rev1=1 rx=0x7f8330013510 tx=0x7f833000b040).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:28:02.981 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:02.979+0000 7f8326ffd700 1 -- 172.21.15.66:0/972155620 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f8330009320 con 0x7f83340f5410 2023-11-17T05:28:02.981 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:02.979+0000 7f833c336700 1 -- 172.21.15.66:0/972155620 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f8334060440 con 0x7f83340f5410 2023-11-17T05:28:02.981 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:02.979+0000 7f8326ffd700 1 -- 172.21.15.66:0/972155620 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(25 keys) v1 ==== 1074+0+0 (secure 0 0 0) 0x7f8330013f00 con 0x7f83340f5410 2023-11-17T05:28:02.981 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:02.979+0000 7f8326ffd700 1 -- 172.21.15.66:0/972155620 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f8330014d90 con 0x7f83340f5410 2023-11-17T05:28:02.981 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:02.979+0000 7f833c336700 1 -- 172.21.15.66:0/972155620 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f8334109ae0 con 0x7f83340f5410 2023-11-17T05:28:02.981 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:02.980+0000 7f8326ffd700 1 -- 172.21.15.66:0/972155620 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 17) v1 ==== 36269+0+0 (secure 0 0 0) 0x7f83300095f0 con 0x7f83340f5410 2023-11-17T05:28:02.982 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:02.980+0000 7f833c336700 1 -- 172.21.15.66:0/972155620 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f8334045ad0 con 0x7f83340f5410 2023-11-17T05:28:02.982 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:02.980+0000 7f8326ffd700 1 --2- 172.21.15.66:0/972155620 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f832002ff60 0x7f8320032420 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:28:02.982 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:02.980+0000 7f8326ffd700 1 -- 172.21.15.66:0/972155620 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(5..5 src has 1..5) v4 ==== 1270+0+0 (secure 0 0 0) 0x7f8330041000 con 0x7f83340f5410 2023-11-17T05:28:02.982 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:02.981+0000 7f83398d1700 1 --2- 172.21.15.66:0/972155620 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f832002ff60 0x7f8320032420 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:28:02.982 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:02.981+0000 7f83398d1700 1 --2- 172.21.15.66:0/972155620 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f832002ff60 0x7f8320032420 secure :-1 s=READY pgs=86 cs=0 l=1 rev1=1 rx=0x7f83280096a0 tx=0x7f8328006b40).ready entity=mgr.14164 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:28:02.991 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:02.985+0000 7f8326ffd700 1 -- 172.21.15.66:0/972155620 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+152212 (secure 0 0 0) 0x7f8330014420 con 0x7f83340f5410 2023-11-17T05:28:03.205 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:03.205+0000 7f833c336700 1 -- 172.21.15.66:0/972155620 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "mon dump", "format": "json"} v 0) v1 -- 0x7f83340fae60 con 0x7f83340f5410 2023-11-17T05:28:03.205 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:03.205+0000 7f8326ffd700 1 -- 172.21.15.66:0/972155620 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mon_command_ack([{"prefix": "mon dump", "format": "json"}]=0 dumped monmap epoch 1 v1) v1 ==== 95+0+690 (secure 0 0 0) 0x7f833001d7f0 con 0x7f83340f5410 2023-11-17T05:28:03.206 INFO:teuthology.orchestra.run.smithi066.stdout: 2023-11-17T05:28:03.206 INFO:teuthology.orchestra.run.smithi066.stdout:{"epoch":1,"fsid":"ed6470e2-8508-11ee-8dbe-212e2dc638e7","modified":"2023-11-17T05:20:13.934038Z","created":"2023-11-17T05:20:13.934038Z","min_mon_release":16,"min_mon_release_name":"pacific","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging"],"optional":[]},"mons":[{"rank":0,"name":"smithi040","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:3300","nonce":0},{"type":"v1","addr":"172.21.15.40:6789","nonce":0}]},"addr":"172.21.15.40:6789/0","public_addr":"172.21.15.40:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2023-11-17T05:28:03.207 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:03.207+0000 7f833c336700 1 -- 172.21.15.66:0/972155620 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f832002ff60 msgr2=0x7f8320032420 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:28:03.207 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:03.207+0000 7f833c336700 1 --2- 172.21.15.66:0/972155620 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f832002ff60 0x7f8320032420 secure :-1 s=READY pgs=86 cs=0 l=1 rev1=1 rx=0x7f83280096a0 tx=0x7f8328006b40).stop 2023-11-17T05:28:03.207 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:03.207+0000 7f833c336700 1 -- 172.21.15.66:0/972155620 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f83340f5410 msgr2=0x7f8334109430 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:28:03.207 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:03.207+0000 7f833c336700 1 --2- 172.21.15.66:0/972155620 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f83340f5410 0x7f8334109430 secure :-1 s=READY pgs=242 cs=0 l=1 rev1=1 rx=0x7f8330013510 tx=0x7f833000b040).stop 2023-11-17T05:28:03.207 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:03.207+0000 7f833c336700 1 -- 172.21.15.66:0/972155620 shutdown_connections 2023-11-17T05:28:03.208 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:03.207+0000 7f833c336700 1 --2- 172.21.15.66:0/972155620 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f832002ff60 0x7f8320032420 unknown :-1 s=CLOSED pgs=86 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:28:03.208 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:03.207+0000 7f833c336700 1 --2- 172.21.15.66:0/972155620 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f83340f5410 0x7f8334109430 unknown :-1 s=CLOSED pgs=242 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:28:03.208 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:03.207+0000 7f833c336700 1 -- 172.21.15.66:0/972155620 >> 172.21.15.66:0/972155620 conn(0x7f83340f0b70 msgr2=0x7f83340f1660 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:28:03.208 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:03.207+0000 7f833c336700 1 -- 172.21.15.66:0/972155620 shutdown_connections 2023-11-17T05:28:03.208 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:03.207+0000 7f833c336700 1 -- 172.21.15.66:0/972155620 wait complete. 2023-11-17T05:28:03.208 INFO:teuthology.orchestra.run.smithi066.stderr:dumped monmap epoch 1 2023-11-17T05:28:03.739 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:28:03 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:28:03.205907+0000 mon.smithi040 (mon.0) 215 : audit [DBG] from='client.? 172.21.15.66:0/972155620' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-11-17T05:28:04.587 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2023-11-17T05:28:04.587 DEBUG:teuthology.orchestra.run.smithi066:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.4 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ed6470e2-8508-11ee-8dbe-212e2dc638e7 -- ceph mon dump -f json 2023-11-17T05:28:04.988 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:28:04 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:28:03.223844+0000 mgr.smithi040.ogcprl (mgr.14164) 212 : cluster [DBG] pgmap v200: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:28:05.730 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:05.728+0000 7f109282c700 1 -- 172.21.15.66:0/2547794911 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f108c0f69d0 msgr2=0x7f108c0f6df0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:28:05.730 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:05.728+0000 7f109282c700 1 --2- 172.21.15.66:0/2547794911 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f108c0f69d0 0x7f108c0f6df0 secure :-1 s=READY pgs=243 cs=0 l=1 rev1=1 rx=0x7f107c009fb0 tx=0x7f107c009b30).stop 2023-11-17T05:28:05.730 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:05.728+0000 7f109282c700 1 -- 172.21.15.66:0/2547794911 shutdown_connections 2023-11-17T05:28:05.730 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:05.728+0000 7f109282c700 1 --2- 172.21.15.66:0/2547794911 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f108c0f69d0 0x7f108c0f6df0 unknown :-1 s=CLOSED pgs=243 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:28:05.730 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:05.728+0000 7f109282c700 1 -- 172.21.15.66:0/2547794911 >> 172.21.15.66:0/2547794911 conn(0x7f108c0f02e0 msgr2=0x7f108c0f2700 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:28:05.730 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:05.728+0000 7f109282c700 1 -- 172.21.15.66:0/2547794911 shutdown_connections 2023-11-17T05:28:05.730 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:05.728+0000 7f109282c700 1 -- 172.21.15.66:0/2547794911 wait complete. 2023-11-17T05:28:05.730 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:05.729+0000 7f109282c700 1 Processor -- start 2023-11-17T05:28:05.730 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:05.729+0000 7f109282c700 1 -- start start 2023-11-17T05:28:05.731 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:05.729+0000 7f109282c700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f108c0f69d0 0x7f108c1066b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:28:05.731 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:05.729+0000 7f109282c700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f108c106ba0 con 0x7f108c0f69d0 2023-11-17T05:28:05.731 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:05.730+0000 7f108bfff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f108c0f69d0 0x7f108c1066b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:28:05.731 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:05.730+0000 7f108bfff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f108c0f69d0 0x7f108c1066b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.66:50518/0 (socket says 172.21.15.66:50518) 2023-11-17T05:28:05.731 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:05.730+0000 7f108bfff700 1 -- 172.21.15.66:0/2890953314 learned_addr learned my addr 172.21.15.66:0/2890953314 (peer_addr_for_me v2:172.21.15.66:0/0) 2023-11-17T05:28:05.732 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:05.730+0000 7f108bfff700 1 -- 172.21.15.66:0/2890953314 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f107c005040 con 0x7f108c0f69d0 2023-11-17T05:28:05.732 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:05.730+0000 7f108bfff700 1 --2- 172.21.15.66:0/2890953314 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f108c0f69d0 0x7f108c1066b0 secure :-1 s=READY pgs=244 cs=0 l=1 rev1=1 rx=0x7f107c013510 tx=0x7f107c00b040).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:28:05.732 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:05.730+0000 7f1088ff9700 1 -- 172.21.15.66:0/2890953314 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f107c009320 con 0x7f108c0f69d0 2023-11-17T05:28:05.732 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:05.730+0000 7f109282c700 1 -- 172.21.15.66:0/2890953314 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f108c106da0 con 0x7f108c0f69d0 2023-11-17T05:28:05.733 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:05.730+0000 7f1088ff9700 1 -- 172.21.15.66:0/2890953314 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(25 keys) v1 ==== 1074+0+0 (secure 0 0 0) 0x7f107c013f00 con 0x7f108c0f69d0 2023-11-17T05:28:05.733 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:05.730+0000 7f1088ff9700 1 -- 172.21.15.66:0/2890953314 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f107c014d90 con 0x7f108c0f69d0 2023-11-17T05:28:05.733 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:05.730+0000 7f109282c700 1 -- 172.21.15.66:0/2890953314 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f108c107230 con 0x7f108c0f69d0 2023-11-17T05:28:05.733 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:05.731+0000 7f1088ff9700 1 -- 172.21.15.66:0/2890953314 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 17) v1 ==== 36269+0+0 (secure 0 0 0) 0x7f107c0263a0 con 0x7f108c0f69d0 2023-11-17T05:28:05.733 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:05.731+0000 7f1088ff9700 1 --2- 172.21.15.66:0/2890953314 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f107402ff60 0x7f1074032420 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:28:05.733 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:05.731+0000 7f1088ff9700 1 -- 172.21.15.66:0/2890953314 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(5..5 src has 1..5) v4 ==== 1270+0+0 (secure 0 0 0) 0x7f107c041080 con 0x7f108c0f69d0 2023-11-17T05:28:05.733 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:05.731+0000 7f109282c700 1 -- 172.21.15.66:0/2890953314 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f108c045ad0 con 0x7f108c0f69d0 2023-11-17T05:28:05.733 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:05.731+0000 7f108b7fe700 1 --2- 172.21.15.66:0/2890953314 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f107402ff60 0x7f1074032420 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:28:05.733 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:05.732+0000 7f108b7fe700 1 --2- 172.21.15.66:0/2890953314 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f107402ff60 0x7f1074032420 secure :-1 s=READY pgs=87 cs=0 l=1 rev1=1 rx=0x7f10800096a0 tx=0x7f1080006b40).ready entity=mgr.14164 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:28:05.737 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:05.736+0000 7f1088ff9700 1 -- 172.21.15.66:0/2890953314 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+152212 (secure 0 0 0) 0x7f107c012920 con 0x7f108c0f69d0 2023-11-17T05:28:05.955 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:05.955+0000 7f109282c700 1 -- 172.21.15.66:0/2890953314 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "mon dump", "format": "json"} v 0) v1 -- 0x7f108c107f80 con 0x7f108c0f69d0 2023-11-17T05:28:05.956 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:05.956+0000 7f1088ff9700 1 -- 172.21.15.66:0/2890953314 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mon_command_ack([{"prefix": "mon dump", "format": "json"}]=0 dumped monmap epoch 1 v1) v1 ==== 95+0+690 (secure 0 0 0) 0x7f107c02b020 con 0x7f108c0f69d0 2023-11-17T05:28:05.956 INFO:teuthology.orchestra.run.smithi066.stdout: 2023-11-17T05:28:05.956 INFO:teuthology.orchestra.run.smithi066.stdout:{"epoch":1,"fsid":"ed6470e2-8508-11ee-8dbe-212e2dc638e7","modified":"2023-11-17T05:20:13.934038Z","created":"2023-11-17T05:20:13.934038Z","min_mon_release":16,"min_mon_release_name":"pacific","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging"],"optional":[]},"mons":[{"rank":0,"name":"smithi040","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:3300","nonce":0},{"type":"v1","addr":"172.21.15.40:6789","nonce":0}]},"addr":"172.21.15.40:6789/0","public_addr":"172.21.15.40:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2023-11-17T05:28:05.958 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:05.957+0000 7f109282c700 1 -- 172.21.15.66:0/2890953314 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f107402ff60 msgr2=0x7f1074032420 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:28:05.958 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:05.957+0000 7f109282c700 1 --2- 172.21.15.66:0/2890953314 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f107402ff60 0x7f1074032420 secure :-1 s=READY pgs=87 cs=0 l=1 rev1=1 rx=0x7f10800096a0 tx=0x7f1080006b40).stop 2023-11-17T05:28:05.958 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:05.957+0000 7f109282c700 1 -- 172.21.15.66:0/2890953314 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f108c0f69d0 msgr2=0x7f108c1066b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:28:05.958 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:05.957+0000 7f109282c700 1 --2- 172.21.15.66:0/2890953314 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f108c0f69d0 0x7f108c1066b0 secure :-1 s=READY pgs=244 cs=0 l=1 rev1=1 rx=0x7f107c013510 tx=0x7f107c00b040).stop 2023-11-17T05:28:05.959 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:05.957+0000 7f109282c700 1 -- 172.21.15.66:0/2890953314 shutdown_connections 2023-11-17T05:28:05.959 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:05.957+0000 7f109282c700 1 --2- 172.21.15.66:0/2890953314 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f107402ff60 0x7f1074032420 unknown :-1 s=CLOSED pgs=87 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:28:05.959 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:05.957+0000 7f109282c700 1 --2- 172.21.15.66:0/2890953314 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f108c0f69d0 0x7f108c1066b0 unknown :-1 s=CLOSED pgs=244 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:28:05.959 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:05.957+0000 7f109282c700 1 -- 172.21.15.66:0/2890953314 >> 172.21.15.66:0/2890953314 conn(0x7f108c0f02e0 msgr2=0x7f108c0f0d90 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:28:05.959 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:05.957+0000 7f109282c700 1 -- 172.21.15.66:0/2890953314 shutdown_connections 2023-11-17T05:28:05.959 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:05.957+0000 7f109282c700 1 -- 172.21.15.66:0/2890953314 wait complete. 2023-11-17T05:28:05.959 INFO:teuthology.orchestra.run.smithi066.stderr:dumped monmap epoch 1 2023-11-17T05:28:06.989 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:28:06 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:28:05.224312+0000 mgr.smithi040.ogcprl (mgr.14164) 213 : cluster [DBG] pgmap v201: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:28:06.989 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:28:06 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:28:05.956469+0000 mon.smithi040 (mon.0) 216 : audit [DBG] from='client.? 172.21.15.66:0/2890953314' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-11-17T05:28:07.344 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2023-11-17T05:28:07.345 DEBUG:teuthology.orchestra.run.smithi066:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.4 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ed6470e2-8508-11ee-8dbe-212e2dc638e7 -- ceph mon dump -f json 2023-11-17T05:28:08.553 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:08.552+0000 7f8caa501700 1 -- 172.21.15.66:0/3249960826 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f8ca40f5410 msgr2=0x7f8ca40f5830 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:28:08.553 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:08.552+0000 7f8caa501700 1 --2- 172.21.15.66:0/3249960826 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f8ca40f5410 0x7f8ca40f5830 secure :-1 s=READY pgs=245 cs=0 l=1 rev1=1 rx=0x7f8c94009fb0 tx=0x7f8c94009b30).stop 2023-11-17T05:28:08.553 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:08.552+0000 7f8caa501700 1 -- 172.21.15.66:0/3249960826 shutdown_connections 2023-11-17T05:28:08.553 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:08.552+0000 7f8caa501700 1 --2- 172.21.15.66:0/3249960826 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f8ca40f5410 0x7f8ca40f5830 unknown :-1 s=CLOSED pgs=245 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:28:08.553 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:08.552+0000 7f8caa501700 1 -- 172.21.15.66:0/3249960826 >> 172.21.15.66:0/3249960826 conn(0x7f8ca40f0b70 msgr2=0x7f8ca40f2f90 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:28:08.554 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:08.552+0000 7f8caa501700 1 -- 172.21.15.66:0/3249960826 shutdown_connections 2023-11-17T05:28:08.554 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:08.552+0000 7f8caa501700 1 -- 172.21.15.66:0/3249960826 wait complete. 2023-11-17T05:28:08.554 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:08.552+0000 7f8caa501700 1 Processor -- start 2023-11-17T05:28:08.554 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:08.553+0000 7f8caa501700 1 -- start start 2023-11-17T05:28:08.554 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:08.553+0000 7f8caa501700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f8ca40f5410 0x7f8ca40fd3a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:28:08.554 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:08.553+0000 7f8caa501700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f8ca40fb820 con 0x7f8ca40f5410 2023-11-17T05:28:08.554 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:08.553+0000 7f8ca3fff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f8ca40f5410 0x7f8ca40fd3a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:28:08.554 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:08.553+0000 7f8ca3fff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f8ca40f5410 0x7f8ca40fd3a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.66:50540/0 (socket says 172.21.15.66:50540) 2023-11-17T05:28:08.554 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:08.553+0000 7f8ca3fff700 1 -- 172.21.15.66:0/3831433718 learned_addr learned my addr 172.21.15.66:0/3831433718 (peer_addr_for_me v2:172.21.15.66:0/0) 2023-11-17T05:28:08.556 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:08.553+0000 7f8ca3fff700 1 -- 172.21.15.66:0/3831433718 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f8c94005040 con 0x7f8ca40f5410 2023-11-17T05:28:08.556 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:08.553+0000 7f8ca3fff700 1 --2- 172.21.15.66:0/3831433718 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f8ca40f5410 0x7f8ca40fd3a0 secure :-1 s=READY pgs=246 cs=0 l=1 rev1=1 rx=0x7f8c94013510 tx=0x7f8c9400b040).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:28:08.556 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:08.553+0000 7f8ca0ff9700 1 -- 172.21.15.66:0/3831433718 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f8c94009320 con 0x7f8ca40f5410 2023-11-17T05:28:08.556 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:08.554+0000 7f8caa501700 1 -- 172.21.15.66:0/3831433718 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f8ca40fb9c0 con 0x7f8ca40f5410 2023-11-17T05:28:08.556 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:08.554+0000 7f8ca0ff9700 1 -- 172.21.15.66:0/3831433718 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(25 keys) v1 ==== 1074+0+0 (secure 0 0 0) 0x7f8c94013f00 con 0x7f8ca40f5410 2023-11-17T05:28:08.556 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:08.554+0000 7f8ca0ff9700 1 -- 172.21.15.66:0/3831433718 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f8c94014d90 con 0x7f8ca40f5410 2023-11-17T05:28:08.556 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:08.554+0000 7f8caa501700 1 -- 172.21.15.66:0/3831433718 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f8ca40fbe50 con 0x7f8ca40f5410 2023-11-17T05:28:08.556 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:08.555+0000 7f8ca0ff9700 1 -- 172.21.15.66:0/3831433718 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 17) v1 ==== 36269+0+0 (secure 0 0 0) 0x7f8c940095f0 con 0x7f8ca40f5410 2023-11-17T05:28:08.556 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:08.555+0000 7f8caa501700 1 -- 172.21.15.66:0/3831433718 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f8ca4059100 con 0x7f8ca40f5410 2023-11-17T05:28:08.556 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:08.555+0000 7f8ca0ff9700 1 --2- 172.21.15.66:0/3831433718 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f8c8c02ff60 0x7f8c8c032420 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:28:08.557 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:08.555+0000 7f8ca0ff9700 1 -- 172.21.15.66:0/3831433718 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(5..5 src has 1..5) v4 ==== 1270+0+0 (secure 0 0 0) 0x7f8c94040fe0 con 0x7f8ca40f5410 2023-11-17T05:28:08.557 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:08.555+0000 7f8ca37fe700 1 --2- 172.21.15.66:0/3831433718 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f8c8c02ff60 0x7f8c8c032420 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:28:08.557 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:08.555+0000 7f8ca37fe700 1 --2- 172.21.15.66:0/3831433718 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f8c8c02ff60 0x7f8c8c032420 secure :-1 s=READY pgs=88 cs=0 l=1 rev1=1 rx=0x7f8c980096a0 tx=0x7f8c98006b40).ready entity=mgr.14164 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:28:08.560 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:08.559+0000 7f8ca0ff9700 1 -- 172.21.15.66:0/3831433718 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+152212 (secure 0 0 0) 0x7f8c9401d7f0 con 0x7f8ca40f5410 2023-11-17T05:28:08.778 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:08.778+0000 7f8caa501700 1 -- 172.21.15.66:0/3831433718 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "mon dump", "format": "json"} v 0) v1 -- 0x7f8ca40fae60 con 0x7f8ca40f5410 2023-11-17T05:28:08.779 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:08.779+0000 7f8ca0ff9700 1 -- 172.21.15.66:0/3831433718 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mon_command_ack([{"prefix": "mon dump", "format": "json"}]=0 dumped monmap epoch 1 v1) v1 ==== 95+0+690 (secure 0 0 0) 0x7f8c94010230 con 0x7f8ca40f5410 2023-11-17T05:28:08.779 INFO:teuthology.orchestra.run.smithi066.stdout: 2023-11-17T05:28:08.779 INFO:teuthology.orchestra.run.smithi066.stdout:{"epoch":1,"fsid":"ed6470e2-8508-11ee-8dbe-212e2dc638e7","modified":"2023-11-17T05:20:13.934038Z","created":"2023-11-17T05:20:13.934038Z","min_mon_release":16,"min_mon_release_name":"pacific","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging"],"optional":[]},"mons":[{"rank":0,"name":"smithi040","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:3300","nonce":0},{"type":"v1","addr":"172.21.15.40:6789","nonce":0}]},"addr":"172.21.15.40:6789/0","public_addr":"172.21.15.40:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2023-11-17T05:28:08.781 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:08.781+0000 7f8caa501700 1 -- 172.21.15.66:0/3831433718 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f8c8c02ff60 msgr2=0x7f8c8c032420 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:28:08.781 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:08.781+0000 7f8caa501700 1 --2- 172.21.15.66:0/3831433718 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f8c8c02ff60 0x7f8c8c032420 secure :-1 s=READY pgs=88 cs=0 l=1 rev1=1 rx=0x7f8c980096a0 tx=0x7f8c98006b40).stop 2023-11-17T05:28:08.781 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:08.781+0000 7f8caa501700 1 -- 172.21.15.66:0/3831433718 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f8ca40f5410 msgr2=0x7f8ca40fd3a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:28:08.782 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:08.781+0000 7f8caa501700 1 --2- 172.21.15.66:0/3831433718 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f8ca40f5410 0x7f8ca40fd3a0 secure :-1 s=READY pgs=246 cs=0 l=1 rev1=1 rx=0x7f8c94013510 tx=0x7f8c9400b040).stop 2023-11-17T05:28:08.782 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:08.781+0000 7f8caa501700 1 -- 172.21.15.66:0/3831433718 shutdown_connections 2023-11-17T05:28:08.782 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:08.781+0000 7f8caa501700 1 --2- 172.21.15.66:0/3831433718 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f8c8c02ff60 0x7f8c8c032420 unknown :-1 s=CLOSED pgs=88 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:28:08.782 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:08.781+0000 7f8caa501700 1 --2- 172.21.15.66:0/3831433718 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f8ca40f5410 0x7f8ca40fd3a0 unknown :-1 s=CLOSED pgs=246 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:28:08.782 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:08.781+0000 7f8caa501700 1 -- 172.21.15.66:0/3831433718 >> 172.21.15.66:0/3831433718 conn(0x7f8ca40f0b70 msgr2=0x7f8ca405fde0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:28:08.782 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:08.781+0000 7f8caa501700 1 -- 172.21.15.66:0/3831433718 shutdown_connections 2023-11-17T05:28:08.782 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:08.781+0000 7f8caa501700 1 -- 172.21.15.66:0/3831433718 wait complete. 2023-11-17T05:28:08.782 INFO:teuthology.orchestra.run.smithi066.stderr:dumped monmap epoch 1 2023-11-17T05:28:08.989 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:28:08 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:28:07.224764+0000 mgr.smithi040.ogcprl (mgr.14164) 214 : cluster [DBG] pgmap v202: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:28:09.989 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:28:09 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:28:08.779460+0000 mon.smithi040 (mon.0) 217 : audit [DBG] from='client.? 172.21.15.66:0/3831433718' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-11-17T05:28:10.204 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2023-11-17T05:28:10.204 DEBUG:teuthology.orchestra.run.smithi066:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.4 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ed6470e2-8508-11ee-8dbe-212e2dc638e7 -- ceph mon dump -f json 2023-11-17T05:28:10.989 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:28:10 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:28:09.225137+0000 mgr.smithi040.ogcprl (mgr.14164) 215 : cluster [DBG] pgmap v203: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:28:11.344 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:11.343+0000 7f19b412d700 1 -- 172.21.15.66:0/783647933 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f19ac0f4740 msgr2=0x7f19ac0f4b60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:28:11.344 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:11.343+0000 7f19b412d700 1 --2- 172.21.15.66:0/783647933 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f19ac0f4740 0x7f19ac0f4b60 secure :-1 s=READY pgs=247 cs=0 l=1 rev1=1 rx=0x7f19a8009fb0 tx=0x7f19a8009b30).stop 2023-11-17T05:28:11.344 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:11.343+0000 7f19b412d700 1 -- 172.21.15.66:0/783647933 shutdown_connections 2023-11-17T05:28:11.344 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:11.343+0000 7f19b412d700 1 --2- 172.21.15.66:0/783647933 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f19ac0f4740 0x7f19ac0f4b60 unknown :-1 s=CLOSED pgs=247 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:28:11.344 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:11.343+0000 7f19b412d700 1 -- 172.21.15.66:0/783647933 >> 172.21.15.66:0/783647933 conn(0x7f19ac0f0260 msgr2=0x7f19ac0f2680 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:28:11.344 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:11.344+0000 7f19b412d700 1 -- 172.21.15.66:0/783647933 shutdown_connections 2023-11-17T05:28:11.345 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:11.344+0000 7f19b412d700 1 -- 172.21.15.66:0/783647933 wait complete. 2023-11-17T05:28:11.346 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:11.344+0000 7f19b412d700 1 Processor -- start 2023-11-17T05:28:11.346 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:11.344+0000 7f19b412d700 1 -- start start 2023-11-17T05:28:11.346 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:11.344+0000 7f19b412d700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f19ac0f4740 0x7f19ac1066d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:28:11.346 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:11.345+0000 7f19b412d700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f19ac106bc0 con 0x7f19ac0f4740 2023-11-17T05:28:11.346 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:11.345+0000 7f19b1ec9700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f19ac0f4740 0x7f19ac1066d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:28:11.346 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:11.345+0000 7f19b1ec9700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f19ac0f4740 0x7f19ac1066d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.66:50546/0 (socket says 172.21.15.66:50546) 2023-11-17T05:28:11.347 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:11.345+0000 7f19b1ec9700 1 -- 172.21.15.66:0/1267612941 learned_addr learned my addr 172.21.15.66:0/1267612941 (peer_addr_for_me v2:172.21.15.66:0/0) 2023-11-17T05:28:11.347 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:11.345+0000 7f19b1ec9700 1 -- 172.21.15.66:0/1267612941 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f19a8005040 con 0x7f19ac0f4740 2023-11-17T05:28:11.347 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:11.345+0000 7f19b1ec9700 1 --2- 172.21.15.66:0/1267612941 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f19ac0f4740 0x7f19ac1066d0 secure :-1 s=READY pgs=248 cs=0 l=1 rev1=1 rx=0x7f19a8013510 tx=0x7f19a800b040).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:28:11.347 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:11.345+0000 7f199e7fc700 1 -- 172.21.15.66:0/1267612941 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f19a8009320 con 0x7f19ac0f4740 2023-11-17T05:28:11.347 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:11.345+0000 7f19b412d700 1 -- 172.21.15.66:0/1267612941 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f19ac106dc0 con 0x7f19ac0f4740 2023-11-17T05:28:11.347 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:11.345+0000 7f199e7fc700 1 -- 172.21.15.66:0/1267612941 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(25 keys) v1 ==== 1074+0+0 (secure 0 0 0) 0x7f19a8013f00 con 0x7f19ac0f4740 2023-11-17T05:28:11.347 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:11.345+0000 7f199e7fc700 1 -- 172.21.15.66:0/1267612941 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f19a8014d90 con 0x7f19ac0f4740 2023-11-17T05:28:11.347 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:11.345+0000 7f19b412d700 1 -- 172.21.15.66:0/1267612941 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f19ac107250 con 0x7f19ac0f4740 2023-11-17T05:28:11.348 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:11.346+0000 7f199e7fc700 1 -- 172.21.15.66:0/1267612941 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 17) v1 ==== 36269+0+0 (secure 0 0 0) 0x7f19a80095f0 con 0x7f19ac0f4740 2023-11-17T05:28:11.349 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:11.346+0000 7f19b412d700 1 -- 172.21.15.66:0/1267612941 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f19ac045ad0 con 0x7f19ac0f4740 2023-11-17T05:28:11.349 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:11.346+0000 7f199e7fc700 1 --2- 172.21.15.66:0/1267612941 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f199802ff60 0x7f1998032420 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:28:11.349 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:11.346+0000 7f199e7fc700 1 -- 172.21.15.66:0/1267612941 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(5..5 src has 1..5) v4 ==== 1270+0+0 (secure 0 0 0) 0x7f19a80410c0 con 0x7f19ac0f4740 2023-11-17T05:28:11.349 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:11.347+0000 7f19b16c8700 1 --2- 172.21.15.66:0/1267612941 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f199802ff60 0x7f1998032420 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:28:11.349 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:11.347+0000 7f19b16c8700 1 --2- 172.21.15.66:0/1267612941 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f199802ff60 0x7f1998032420 secure :-1 s=READY pgs=89 cs=0 l=1 rev1=1 rx=0x7f19a00096a0 tx=0x7f19a0006b40).ready entity=mgr.14164 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:28:11.351 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:11.351+0000 7f199e7fc700 1 -- 172.21.15.66:0/1267612941 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+152212 (secure 0 0 0) 0x7f19a8014420 con 0x7f19ac0f4740 2023-11-17T05:28:11.569 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:11.569+0000 7f19b412d700 1 -- 172.21.15.66:0/1267612941 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "mon dump", "format": "json"} v 0) v1 -- 0x7f19ac108100 con 0x7f19ac0f4740 2023-11-17T05:28:11.570 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:11.570+0000 7f199e7fc700 1 -- 172.21.15.66:0/1267612941 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mon_command_ack([{"prefix": "mon dump", "format": "json"}]=0 dumped monmap epoch 1 v1) v1 ==== 95+0+690 (secure 0 0 0) 0x7f19a8010230 con 0x7f19ac0f4740 2023-11-17T05:28:11.570 INFO:teuthology.orchestra.run.smithi066.stdout: 2023-11-17T05:28:11.570 INFO:teuthology.orchestra.run.smithi066.stdout:{"epoch":1,"fsid":"ed6470e2-8508-11ee-8dbe-212e2dc638e7","modified":"2023-11-17T05:20:13.934038Z","created":"2023-11-17T05:20:13.934038Z","min_mon_release":16,"min_mon_release_name":"pacific","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging"],"optional":[]},"mons":[{"rank":0,"name":"smithi040","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:3300","nonce":0},{"type":"v1","addr":"172.21.15.40:6789","nonce":0}]},"addr":"172.21.15.40:6789/0","public_addr":"172.21.15.40:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2023-11-17T05:28:11.572 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:11.571+0000 7f19b412d700 1 -- 172.21.15.66:0/1267612941 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f199802ff60 msgr2=0x7f1998032420 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:28:11.572 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:11.571+0000 7f19b412d700 1 --2- 172.21.15.66:0/1267612941 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f199802ff60 0x7f1998032420 secure :-1 s=READY pgs=89 cs=0 l=1 rev1=1 rx=0x7f19a00096a0 tx=0x7f19a0006b40).stop 2023-11-17T05:28:11.572 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:11.571+0000 7f19b412d700 1 -- 172.21.15.66:0/1267612941 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f19ac0f4740 msgr2=0x7f19ac1066d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:28:11.572 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:11.571+0000 7f19b412d700 1 --2- 172.21.15.66:0/1267612941 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f19ac0f4740 0x7f19ac1066d0 secure :-1 s=READY pgs=248 cs=0 l=1 rev1=1 rx=0x7f19a8013510 tx=0x7f19a800b040).stop 2023-11-17T05:28:11.572 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:11.571+0000 7f19b412d700 1 -- 172.21.15.66:0/1267612941 shutdown_connections 2023-11-17T05:28:11.572 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:11.571+0000 7f19b412d700 1 --2- 172.21.15.66:0/1267612941 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f199802ff60 0x7f1998032420 unknown :-1 s=CLOSED pgs=89 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:28:11.572 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:11.571+0000 7f19b412d700 1 --2- 172.21.15.66:0/1267612941 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f19ac0f4740 0x7f19ac1066d0 unknown :-1 s=CLOSED pgs=248 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:28:11.572 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:11.571+0000 7f19b412d700 1 -- 172.21.15.66:0/1267612941 >> 172.21.15.66:0/1267612941 conn(0x7f19ac0f0260 msgr2=0x7f19ac0f0d80 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:28:11.572 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:11.571+0000 7f19b412d700 1 -- 172.21.15.66:0/1267612941 shutdown_connections 2023-11-17T05:28:11.572 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:11.571+0000 7f19b412d700 1 -- 172.21.15.66:0/1267612941 wait complete. 2023-11-17T05:28:11.573 INFO:teuthology.orchestra.run.smithi066.stderr:dumped monmap epoch 1 2023-11-17T05:28:11.989 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:28:11 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:28:11.314438+0000 mon.smithi040 (mon.0) 218 : audit [INF] from='mgr.14164 172.21.15.40:0/2203032451' entity='mgr.smithi040.ogcprl' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi040.ogcprl/mirror_snapshot_schedule"}]: dispatch 2023-11-17T05:28:11.989 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:28:11 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:28:11.315672+0000 mon.smithi040 (mon.0) 219 : audit [INF] from='mgr.14164 172.21.15.40:0/2203032451' entity='mgr.smithi040.ogcprl' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi040.ogcprl/trash_purge_schedule"}]: dispatch 2023-11-17T05:28:12.908 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2023-11-17T05:28:12.909 DEBUG:teuthology.orchestra.run.smithi066:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.4 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ed6470e2-8508-11ee-8dbe-212e2dc638e7 -- ceph mon dump -f json 2023-11-17T05:28:12.989 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:28:12 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:28:11.225519+0000 mgr.smithi040.ogcprl (mgr.14164) 216 : cluster [DBG] pgmap v204: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:28:12.989 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:28:12 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:28:11.570499+0000 mon.smithi040 (mon.0) 220 : audit [DBG] from='client.? 172.21.15.66:0/1267612941' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-11-17T05:28:14.093 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:14.090+0000 7f9b45b31700 1 -- 172.21.15.66:0/1633753324 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f9b400f62a0 msgr2=0x7f9b400f66c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:28:14.093 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:14.090+0000 7f9b45b31700 1 --2- 172.21.15.66:0/1633753324 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f9b400f62a0 0x7f9b400f66c0 secure :-1 s=READY pgs=249 cs=0 l=1 rev1=1 rx=0x7f9b34009fb0 tx=0x7f9b34009b30).stop 2023-11-17T05:28:14.093 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:14.090+0000 7f9b45b31700 1 -- 172.21.15.66:0/1633753324 shutdown_connections 2023-11-17T05:28:14.093 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:14.090+0000 7f9b45b31700 1 --2- 172.21.15.66:0/1633753324 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f9b400f62a0 0x7f9b400f66c0 unknown :-1 s=CLOSED pgs=249 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:28:14.093 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:14.090+0000 7f9b45b31700 1 -- 172.21.15.66:0/1633753324 >> 172.21.15.66:0/1633753324 conn(0x7f9b400f1aa0 msgr2=0x7f9b400f3ec0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:28:14.093 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:14.090+0000 7f9b45b31700 1 -- 172.21.15.66:0/1633753324 shutdown_connections 2023-11-17T05:28:14.093 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:14.090+0000 7f9b45b31700 1 -- 172.21.15.66:0/1633753324 wait complete. 2023-11-17T05:28:14.093 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:14.090+0000 7f9b45b31700 1 Processor -- start 2023-11-17T05:28:14.093 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:14.090+0000 7f9b45b31700 1 -- start start 2023-11-17T05:28:14.093 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:14.091+0000 7f9b45b31700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f9b400f62a0 0x7f9b401064b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:28:14.094 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:14.091+0000 7f9b45b31700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f9b401069a0 con 0x7f9b400f62a0 2023-11-17T05:28:14.104 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:14.091+0000 7f9b44b2f700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f9b400f62a0 0x7f9b401064b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:28:14.105 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:14.091+0000 7f9b44b2f700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f9b400f62a0 0x7f9b401064b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.66:33752/0 (socket says 172.21.15.66:33752) 2023-11-17T05:28:14.105 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:14.091+0000 7f9b44b2f700 1 -- 172.21.15.66:0/2651271673 learned_addr learned my addr 172.21.15.66:0/2651271673 (peer_addr_for_me v2:172.21.15.66:0/0) 2023-11-17T05:28:14.105 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:14.091+0000 7f9b44b2f700 1 -- 172.21.15.66:0/2651271673 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f9b34005040 con 0x7f9b400f62a0 2023-11-17T05:28:14.105 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:14.091+0000 7f9b44b2f700 1 --2- 172.21.15.66:0/2651271673 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f9b400f62a0 0x7f9b401064b0 secure :-1 s=READY pgs=250 cs=0 l=1 rev1=1 rx=0x7f9b34013510 tx=0x7f9b3400b040).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:28:14.105 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:14.092+0000 7f9b3d7fa700 1 -- 172.21.15.66:0/2651271673 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f9b34009320 con 0x7f9b400f62a0 2023-11-17T05:28:14.105 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:14.092+0000 7f9b3d7fa700 1 -- 172.21.15.66:0/2651271673 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(25 keys) v1 ==== 1074+0+0 (secure 0 0 0) 0x7f9b34013f00 con 0x7f9b400f62a0 2023-11-17T05:28:14.105 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:14.092+0000 7f9b45b31700 1 -- 172.21.15.66:0/2651271673 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f9b40106ba0 con 0x7f9b400f62a0 2023-11-17T05:28:14.105 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:14.092+0000 7f9b3d7fa700 1 -- 172.21.15.66:0/2651271673 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f9b34014dc0 con 0x7f9b400f62a0 2023-11-17T05:28:14.105 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:14.092+0000 7f9b45b31700 1 -- 172.21.15.66:0/2651271673 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f9b40107030 con 0x7f9b400f62a0 2023-11-17T05:28:14.105 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:14.092+0000 7f9b3d7fa700 1 -- 172.21.15.66:0/2651271673 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 17) v1 ==== 36269+0+0 (secure 0 0 0) 0x7f9b340095f0 con 0x7f9b400f62a0 2023-11-17T05:28:14.105 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:14.092+0000 7f9b45b31700 1 -- 172.21.15.66:0/2651271673 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f9b40045ad0 con 0x7f9b400f62a0 2023-11-17T05:28:14.105 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:14.093+0000 7f9b3d7fa700 1 --2- 172.21.15.66:0/2651271673 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f9b3002ff60 0x7f9b30032420 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:28:14.105 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:14.093+0000 7f9b3d7fa700 1 -- 172.21.15.66:0/2651271673 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(5..5 src has 1..5) v4 ==== 1270+0+0 (secure 0 0 0) 0x7f9b34041100 con 0x7f9b400f62a0 2023-11-17T05:28:14.106 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:14.093+0000 7f9b3ffff700 1 --2- 172.21.15.66:0/2651271673 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f9b3002ff60 0x7f9b30032420 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:28:14.106 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:14.093+0000 7f9b3ffff700 1 --2- 172.21.15.66:0/2651271673 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f9b3002ff60 0x7f9b30032420 secure :-1 s=READY pgs=90 cs=0 l=1 rev1=1 rx=0x7f9b2c0096a0 tx=0x7f9b2c006b40).ready entity=mgr.14164 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:28:14.106 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:14.098+0000 7f9b3d7fa700 1 -- 172.21.15.66:0/2651271673 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+152212 (secure 0 0 0) 0x7f9b34014420 con 0x7f9b400f62a0 2023-11-17T05:28:14.317 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:14.316+0000 7f9b45b31700 1 -- 172.21.15.66:0/2651271673 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "mon dump", "format": "json"} v 0) v1 -- 0x7f9b40107ee0 con 0x7f9b400f62a0 2023-11-17T05:28:14.317 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:14.316+0000 7f9b3d7fa700 1 -- 172.21.15.66:0/2651271673 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mon_command_ack([{"prefix": "mon dump", "format": "json"}]=0 dumped monmap epoch 1 v1) v1 ==== 95+0+690 (secure 0 0 0) 0x7f9b34010230 con 0x7f9b400f62a0 2023-11-17T05:28:14.321 INFO:teuthology.orchestra.run.smithi066.stdout: 2023-11-17T05:28:14.321 INFO:teuthology.orchestra.run.smithi066.stdout:{"epoch":1,"fsid":"ed6470e2-8508-11ee-8dbe-212e2dc638e7","modified":"2023-11-17T05:20:13.934038Z","created":"2023-11-17T05:20:13.934038Z","min_mon_release":16,"min_mon_release_name":"pacific","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging"],"optional":[]},"mons":[{"rank":0,"name":"smithi040","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:3300","nonce":0},{"type":"v1","addr":"172.21.15.40:6789","nonce":0}]},"addr":"172.21.15.40:6789/0","public_addr":"172.21.15.40:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2023-11-17T05:28:14.325 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:14.318+0000 7f9b45b31700 1 -- 172.21.15.66:0/2651271673 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f9b3002ff60 msgr2=0x7f9b30032420 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:28:14.326 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:14.318+0000 7f9b45b31700 1 --2- 172.21.15.66:0/2651271673 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f9b3002ff60 0x7f9b30032420 secure :-1 s=READY pgs=90 cs=0 l=1 rev1=1 rx=0x7f9b2c0096a0 tx=0x7f9b2c006b40).stop 2023-11-17T05:28:14.327 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:14.318+0000 7f9b45b31700 1 -- 172.21.15.66:0/2651271673 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f9b400f62a0 msgr2=0x7f9b401064b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:28:14.327 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:14.318+0000 7f9b45b31700 1 --2- 172.21.15.66:0/2651271673 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f9b400f62a0 0x7f9b401064b0 secure :-1 s=READY pgs=250 cs=0 l=1 rev1=1 rx=0x7f9b34013510 tx=0x7f9b3400b040).stop 2023-11-17T05:28:14.329 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:14.318+0000 7f9b45b31700 1 -- 172.21.15.66:0/2651271673 shutdown_connections 2023-11-17T05:28:14.329 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:14.318+0000 7f9b45b31700 1 --2- 172.21.15.66:0/2651271673 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f9b3002ff60 0x7f9b30032420 unknown :-1 s=CLOSED pgs=90 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:28:14.330 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:14.318+0000 7f9b45b31700 1 --2- 172.21.15.66:0/2651271673 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f9b400f62a0 0x7f9b401064b0 unknown :-1 s=CLOSED pgs=250 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:28:14.330 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:14.318+0000 7f9b45b31700 1 -- 172.21.15.66:0/2651271673 >> 172.21.15.66:0/2651271673 conn(0x7f9b400f1aa0 msgr2=0x7f9b400f2510 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:28:14.331 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:14.318+0000 7f9b45b31700 1 -- 172.21.15.66:0/2651271673 shutdown_connections 2023-11-17T05:28:14.331 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:14.318+0000 7f9b45b31700 1 -- 172.21.15.66:0/2651271673 wait complete. 2023-11-17T05:28:14.332 INFO:teuthology.orchestra.run.smithi066.stderr:dumped monmap epoch 1 2023-11-17T05:28:14.989 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:28:14 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:28:13.225851+0000 mgr.smithi040.ogcprl (mgr.14164) 217 : cluster [DBG] pgmap v205: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:28:14.989 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:28:14 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:28:14.317065+0000 mon.smithi040 (mon.0) 221 : audit [DBG] from='client.? 172.21.15.66:0/2651271673' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-11-17T05:28:15.706 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2023-11-17T05:28:15.706 DEBUG:teuthology.orchestra.run.smithi066:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.4 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ed6470e2-8508-11ee-8dbe-212e2dc638e7 -- ceph mon dump -f json 2023-11-17T05:28:16.921 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:16.920+0000 7f9f7f6c6700 1 -- 172.21.15.66:0/1045850670 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f9f780f6510 msgr2=0x7f9f780f6930 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:28:16.922 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:16.920+0000 7f9f7f6c6700 1 --2- 172.21.15.66:0/1045850670 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f9f780f6510 0x7f9f780f6930 secure :-1 s=READY pgs=251 cs=0 l=1 rev1=1 rx=0x7f9f68009fb0 tx=0x7f9f68009b30).stop 2023-11-17T05:28:16.922 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:16.920+0000 7f9f7f6c6700 1 -- 172.21.15.66:0/1045850670 shutdown_connections 2023-11-17T05:28:16.922 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:16.920+0000 7f9f7f6c6700 1 --2- 172.21.15.66:0/1045850670 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f9f780f6510 0x7f9f780f6930 unknown :-1 s=CLOSED pgs=251 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:28:16.922 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:16.920+0000 7f9f7f6c6700 1 -- 172.21.15.66:0/1045850670 >> 172.21.15.66:0/1045850670 conn(0x7f9f780f1c70 msgr2=0x7f9f780f4090 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:28:16.922 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:16.921+0000 7f9f7f6c6700 1 -- 172.21.15.66:0/1045850670 shutdown_connections 2023-11-17T05:28:16.922 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:16.921+0000 7f9f7f6c6700 1 -- 172.21.15.66:0/1045850670 wait complete. 2023-11-17T05:28:16.922 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:16.921+0000 7f9f7f6c6700 1 Processor -- start 2023-11-17T05:28:16.922 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:16.921+0000 7f9f7f6c6700 1 -- start start 2023-11-17T05:28:16.924 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:16.922+0000 7f9f7f6c6700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f9f780f6510 0x7f9f78106730 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:28:16.924 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:16.922+0000 7f9f7f6c6700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f9f78106c20 con 0x7f9f780f6510 2023-11-17T05:28:16.924 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:16.922+0000 7f9f7d462700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f9f780f6510 0x7f9f78106730 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:28:16.924 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:16.922+0000 7f9f7d462700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f9f780f6510 0x7f9f78106730 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.66:33760/0 (socket says 172.21.15.66:33760) 2023-11-17T05:28:16.924 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:16.922+0000 7f9f7d462700 1 -- 172.21.15.66:0/2017795917 learned_addr learned my addr 172.21.15.66:0/2017795917 (peer_addr_for_me v2:172.21.15.66:0/0) 2023-11-17T05:28:16.924 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:16.922+0000 7f9f7d462700 1 -- 172.21.15.66:0/2017795917 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f9f68005040 con 0x7f9f780f6510 2023-11-17T05:28:16.925 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:16.922+0000 7f9f7d462700 1 --2- 172.21.15.66:0/2017795917 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f9f780f6510 0x7f9f78106730 secure :-1 s=READY pgs=252 cs=0 l=1 rev1=1 rx=0x7f9f68013510 tx=0x7f9f6800b040).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:28:16.925 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:16.923+0000 7f9f6dffb700 1 -- 172.21.15.66:0/2017795917 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f9f68009320 con 0x7f9f780f6510 2023-11-17T05:28:16.925 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:16.923+0000 7f9f7f6c6700 1 -- 172.21.15.66:0/2017795917 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f9f78106e20 con 0x7f9f780f6510 2023-11-17T05:28:16.925 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:16.923+0000 7f9f6dffb700 1 -- 172.21.15.66:0/2017795917 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(25 keys) v1 ==== 1074+0+0 (secure 0 0 0) 0x7f9f68013f00 con 0x7f9f780f6510 2023-11-17T05:28:16.925 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:16.923+0000 7f9f6dffb700 1 -- 172.21.15.66:0/2017795917 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f9f68014d90 con 0x7f9f780f6510 2023-11-17T05:28:16.925 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:16.923+0000 7f9f7f6c6700 1 -- 172.21.15.66:0/2017795917 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f9f781072b0 con 0x7f9f780f6510 2023-11-17T05:28:16.925 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:16.924+0000 7f9f6dffb700 1 -- 172.21.15.66:0/2017795917 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 17) v1 ==== 36269+0+0 (secure 0 0 0) 0x7f9f680263a0 con 0x7f9f780f6510 2023-11-17T05:28:16.925 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:16.924+0000 7f9f6dffb700 1 --2- 172.21.15.66:0/2017795917 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f9f6402ff60 0x7f9f64032420 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:28:16.925 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:16.924+0000 7f9f6dffb700 1 -- 172.21.15.66:0/2017795917 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(5..5 src has 1..5) v4 ==== 1270+0+0 (secure 0 0 0) 0x7f9f68041f10 con 0x7f9f780f6510 2023-11-17T05:28:16.926 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:16.924+0000 7f9f7f6c6700 1 -- 172.21.15.66:0/2017795917 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f9f78045ad0 con 0x7f9f780f6510 2023-11-17T05:28:16.926 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:16.924+0000 7f9f7cc61700 1 --2- 172.21.15.66:0/2017795917 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f9f6402ff60 0x7f9f64032420 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:28:16.926 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:16.924+0000 7f9f7cc61700 1 --2- 172.21.15.66:0/2017795917 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f9f6402ff60 0x7f9f64032420 secure :-1 s=READY pgs=91 cs=0 l=1 rev1=1 rx=0x7f9f740096a0 tx=0x7f9f74006b40).ready entity=mgr.14164 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:28:16.929 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:16.929+0000 7f9f6dffb700 1 -- 172.21.15.66:0/2017795917 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+152212 (secure 0 0 0) 0x7f9f68012920 con 0x7f9f780f6510 2023-11-17T05:28:16.989 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:28:16 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:28:15.226180+0000 mgr.smithi040.ogcprl (mgr.14164) 218 : cluster [DBG] pgmap v206: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:28:17.146 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:17.146+0000 7f9f7f6c6700 1 -- 172.21.15.66:0/2017795917 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "mon dump", "format": "json"} v 0) v1 -- 0x7f9f78107fd0 con 0x7f9f780f6510 2023-11-17T05:28:17.147 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:17.147+0000 7f9f6dffb700 1 -- 172.21.15.66:0/2017795917 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mon_command_ack([{"prefix": "mon dump", "format": "json"}]=0 dumped monmap epoch 1 v1) v1 ==== 95+0+690 (secure 0 0 0) 0x7f9f6802b020 con 0x7f9f780f6510 2023-11-17T05:28:17.147 INFO:teuthology.orchestra.run.smithi066.stdout: 2023-11-17T05:28:17.147 INFO:teuthology.orchestra.run.smithi066.stdout:{"epoch":1,"fsid":"ed6470e2-8508-11ee-8dbe-212e2dc638e7","modified":"2023-11-17T05:20:13.934038Z","created":"2023-11-17T05:20:13.934038Z","min_mon_release":16,"min_mon_release_name":"pacific","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging"],"optional":[]},"mons":[{"rank":0,"name":"smithi040","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:3300","nonce":0},{"type":"v1","addr":"172.21.15.40:6789","nonce":0}]},"addr":"172.21.15.40:6789/0","public_addr":"172.21.15.40:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2023-11-17T05:28:17.149 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:17.148+0000 7f9f7f6c6700 1 -- 172.21.15.66:0/2017795917 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f9f6402ff60 msgr2=0x7f9f64032420 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:28:17.149 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:17.148+0000 7f9f7f6c6700 1 --2- 172.21.15.66:0/2017795917 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f9f6402ff60 0x7f9f64032420 secure :-1 s=READY pgs=91 cs=0 l=1 rev1=1 rx=0x7f9f740096a0 tx=0x7f9f74006b40).stop 2023-11-17T05:28:17.149 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:17.148+0000 7f9f7f6c6700 1 -- 172.21.15.66:0/2017795917 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f9f780f6510 msgr2=0x7f9f78106730 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:28:17.149 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:17.148+0000 7f9f7f6c6700 1 --2- 172.21.15.66:0/2017795917 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f9f780f6510 0x7f9f78106730 secure :-1 s=READY pgs=252 cs=0 l=1 rev1=1 rx=0x7f9f68013510 tx=0x7f9f6800b040).stop 2023-11-17T05:28:17.149 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:17.149+0000 7f9f7f6c6700 1 -- 172.21.15.66:0/2017795917 shutdown_connections 2023-11-17T05:28:17.149 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:17.149+0000 7f9f7f6c6700 1 --2- 172.21.15.66:0/2017795917 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f9f6402ff60 0x7f9f64032420 unknown :-1 s=CLOSED pgs=91 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:28:17.149 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:17.149+0000 7f9f7f6c6700 1 --2- 172.21.15.66:0/2017795917 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f9f780f6510 0x7f9f78106730 unknown :-1 s=CLOSED pgs=252 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:28:17.149 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:17.149+0000 7f9f7f6c6700 1 -- 172.21.15.66:0/2017795917 >> 172.21.15.66:0/2017795917 conn(0x7f9f780f1c70 msgr2=0x7f9f780f2760 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:28:17.150 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:17.149+0000 7f9f7f6c6700 1 -- 172.21.15.66:0/2017795917 shutdown_connections 2023-11-17T05:28:17.150 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:17.149+0000 7f9f7f6c6700 1 -- 172.21.15.66:0/2017795917 wait complete. 2023-11-17T05:28:17.150 INFO:teuthology.orchestra.run.smithi066.stderr:dumped monmap epoch 1 2023-11-17T05:28:17.990 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:28:17 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:28:17.147524+0000 mon.smithi040 (mon.0) 222 : audit [DBG] from='client.? 172.21.15.66:0/2017795917' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-11-17T05:28:18.567 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2023-11-17T05:28:18.568 DEBUG:teuthology.orchestra.run.smithi066:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.4 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ed6470e2-8508-11ee-8dbe-212e2dc638e7 -- ceph mon dump -f json 2023-11-17T05:28:18.989 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:28:18 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:28:17.226463+0000 mgr.smithi040.ogcprl (mgr.14164) 219 : cluster [DBG] pgmap v207: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:28:19.726 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:19.726+0000 7faef2c6b700 1 -- 172.21.15.66:0/2725848480 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7faeec0601e0 msgr2=0x7faeec0f8f20 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:28:19.727 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:19.726+0000 7faef2c6b700 1 --2- 172.21.15.66:0/2725848480 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7faeec0601e0 0x7faeec0f8f20 secure :-1 s=READY pgs=253 cs=0 l=1 rev1=1 rx=0x7faee8009fb0 tx=0x7faee8009b30).stop 2023-11-17T05:28:19.727 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:19.726+0000 7faef2c6b700 1 -- 172.21.15.66:0/2725848480 shutdown_connections 2023-11-17T05:28:19.727 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:19.726+0000 7faef2c6b700 1 --2- 172.21.15.66:0/2725848480 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7faeec0601e0 0x7faeec0f8f20 unknown :-1 s=CLOSED pgs=253 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:28:19.727 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:19.726+0000 7faef2c6b700 1 -- 172.21.15.66:0/2725848480 >> 172.21.15.66:0/2725848480 conn(0x7faeec0f09a0 msgr2=0x7faeec0f2dc0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:28:19.727 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:19.726+0000 7faef2c6b700 1 -- 172.21.15.66:0/2725848480 shutdown_connections 2023-11-17T05:28:19.727 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:19.726+0000 7faef2c6b700 1 -- 172.21.15.66:0/2725848480 wait complete. 2023-11-17T05:28:19.728 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:19.727+0000 7faef2c6b700 1 Processor -- start 2023-11-17T05:28:19.728 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:19.727+0000 7faef2c6b700 1 -- start start 2023-11-17T05:28:19.729 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:19.727+0000 7faef2c6b700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7faeec0601e0 0x7faeec0f69a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:28:19.729 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:19.727+0000 7faef2c6b700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7faeec0f4e20 con 0x7faeec0601e0 2023-11-17T05:28:19.729 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:19.727+0000 7faef1c69700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7faeec0601e0 0x7faeec0f69a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:28:19.729 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:19.727+0000 7faef1c69700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7faeec0601e0 0x7faeec0f69a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.66:33778/0 (socket says 172.21.15.66:33778) 2023-11-17T05:28:19.729 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:19.727+0000 7faef1c69700 1 -- 172.21.15.66:0/1542807274 learned_addr learned my addr 172.21.15.66:0/1542807274 (peer_addr_for_me v2:172.21.15.66:0/0) 2023-11-17T05:28:19.729 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:19.727+0000 7faef1c69700 1 -- 172.21.15.66:0/1542807274 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7faee8005040 con 0x7faeec0601e0 2023-11-17T05:28:19.730 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:19.728+0000 7faef1c69700 1 --2- 172.21.15.66:0/1542807274 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7faeec0601e0 0x7faeec0f69a0 secure :-1 s=READY pgs=254 cs=0 l=1 rev1=1 rx=0x7faee8013510 tx=0x7faee800b040).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:28:19.730 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:19.728+0000 7faee27fc700 1 -- 172.21.15.66:0/1542807274 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7faee8009320 con 0x7faeec0601e0 2023-11-17T05:28:19.730 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:19.728+0000 7faef2c6b700 1 -- 172.21.15.66:0/1542807274 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7faeec0f4fc0 con 0x7faeec0601e0 2023-11-17T05:28:19.730 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:19.728+0000 7faee27fc700 1 -- 172.21.15.66:0/1542807274 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(25 keys) v1 ==== 1074+0+0 (secure 0 0 0) 0x7faee8013f00 con 0x7faeec0601e0 2023-11-17T05:28:19.730 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:19.728+0000 7faee27fc700 1 -- 172.21.15.66:0/1542807274 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7faee8014d90 con 0x7faeec0601e0 2023-11-17T05:28:19.730 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:19.728+0000 7faef2c6b700 1 -- 172.21.15.66:0/1542807274 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7faeec0f5450 con 0x7faeec0601e0 2023-11-17T05:28:19.731 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:19.729+0000 7faef2c6b700 1 -- 172.21.15.66:0/1542807274 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7faeec059100 con 0x7faeec0601e0 2023-11-17T05:28:19.731 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:19.729+0000 7faee27fc700 1 -- 172.21.15.66:0/1542807274 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 17) v1 ==== 36269+0+0 (secure 0 0 0) 0x7faee80095f0 con 0x7faeec0601e0 2023-11-17T05:28:19.732 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:19.729+0000 7faee27fc700 1 --2- 172.21.15.66:0/1542807274 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7faed802ff60 0x7faed8032420 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:28:19.732 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:19.729+0000 7faee27fc700 1 -- 172.21.15.66:0/1542807274 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(5..5 src has 1..5) v4 ==== 1270+0+0 (secure 0 0 0) 0x7faee80410a0 con 0x7faeec0601e0 2023-11-17T05:28:19.732 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:19.729+0000 7faef1468700 1 --2- 172.21.15.66:0/1542807274 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7faed802ff60 0x7faed8032420 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:28:19.732 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:19.729+0000 7faef1468700 1 --2- 172.21.15.66:0/1542807274 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7faed802ff60 0x7faed8032420 secure :-1 s=READY pgs=92 cs=0 l=1 rev1=1 rx=0x7faedc0096a0 tx=0x7faedc006b40).ready entity=mgr.14164 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:28:19.736 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:19.734+0000 7faee27fc700 1 -- 172.21.15.66:0/1542807274 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+152212 (secure 0 0 0) 0x7faee801d7f0 con 0x7faeec0601e0 2023-11-17T05:28:19.955 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:19.955+0000 7faef2c6b700 1 -- 172.21.15.66:0/1542807274 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "mon dump", "format": "json"} v 0) v1 -- 0x7faeec0fbcd0 con 0x7faeec0601e0 2023-11-17T05:28:19.956 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:19.956+0000 7faee27fc700 1 -- 172.21.15.66:0/1542807274 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mon_command_ack([{"prefix": "mon dump", "format": "json"}]=0 dumped monmap epoch 1 v1) v1 ==== 95+0+690 (secure 0 0 0) 0x7faee8010230 con 0x7faeec0601e0 2023-11-17T05:28:19.956 INFO:teuthology.orchestra.run.smithi066.stdout: 2023-11-17T05:28:19.956 INFO:teuthology.orchestra.run.smithi066.stdout:{"epoch":1,"fsid":"ed6470e2-8508-11ee-8dbe-212e2dc638e7","modified":"2023-11-17T05:20:13.934038Z","created":"2023-11-17T05:20:13.934038Z","min_mon_release":16,"min_mon_release_name":"pacific","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging"],"optional":[]},"mons":[{"rank":0,"name":"smithi040","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:3300","nonce":0},{"type":"v1","addr":"172.21.15.40:6789","nonce":0}]},"addr":"172.21.15.40:6789/0","public_addr":"172.21.15.40:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2023-11-17T05:28:19.958 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:19.957+0000 7faef2c6b700 1 -- 172.21.15.66:0/1542807274 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7faed802ff60 msgr2=0x7faed8032420 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:28:19.958 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:19.957+0000 7faef2c6b700 1 --2- 172.21.15.66:0/1542807274 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7faed802ff60 0x7faed8032420 secure :-1 s=READY pgs=92 cs=0 l=1 rev1=1 rx=0x7faedc0096a0 tx=0x7faedc006b40).stop 2023-11-17T05:28:19.958 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:19.958+0000 7faef2c6b700 1 -- 172.21.15.66:0/1542807274 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7faeec0601e0 msgr2=0x7faeec0f69a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:28:19.958 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:19.958+0000 7faef2c6b700 1 --2- 172.21.15.66:0/1542807274 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7faeec0601e0 0x7faeec0f69a0 secure :-1 s=READY pgs=254 cs=0 l=1 rev1=1 rx=0x7faee8013510 tx=0x7faee800b040).stop 2023-11-17T05:28:19.958 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:19.958+0000 7faef2c6b700 1 -- 172.21.15.66:0/1542807274 shutdown_connections 2023-11-17T05:28:19.958 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:19.958+0000 7faef2c6b700 1 --2- 172.21.15.66:0/1542807274 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7faed802ff60 0x7faed8032420 unknown :-1 s=CLOSED pgs=92 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:28:19.958 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:19.958+0000 7faef2c6b700 1 --2- 172.21.15.66:0/1542807274 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7faeec0601e0 0x7faeec0f69a0 unknown :-1 s=CLOSED pgs=254 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:28:19.958 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:19.958+0000 7faef2c6b700 1 -- 172.21.15.66:0/1542807274 >> 172.21.15.66:0/1542807274 conn(0x7faeec0f09a0 msgr2=0x7faeec0f1420 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:28:19.958 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:19.958+0000 7faef2c6b700 1 -- 172.21.15.66:0/1542807274 shutdown_connections 2023-11-17T05:28:19.959 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:19.958+0000 7faef2c6b700 1 -- 172.21.15.66:0/1542807274 wait complete. 2023-11-17T05:28:19.959 INFO:teuthology.orchestra.run.smithi066.stderr:dumped monmap epoch 1 2023-11-17T05:28:20.989 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:28:20 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:28:19.226857+0000 mgr.smithi040.ogcprl (mgr.14164) 220 : cluster [DBG] pgmap v208: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:28:20.989 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:28:20 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:28:19.956462+0000 mon.smithi040 (mon.0) 223 : audit [DBG] from='client.? 172.21.15.66:0/1542807274' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-11-17T05:28:21.353 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2023-11-17T05:28:21.353 DEBUG:teuthology.orchestra.run.smithi066:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.4 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ed6470e2-8508-11ee-8dbe-212e2dc638e7 -- ceph mon dump -f json 2023-11-17T05:28:22.535 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:22.532+0000 7fc984acc700 1 -- 172.21.15.66:0/2414588887 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fc9800f6510 msgr2=0x7fc9800f6930 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:28:22.535 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:22.532+0000 7fc984acc700 1 --2- 172.21.15.66:0/2414588887 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fc9800f6510 0x7fc9800f6930 secure :-1 s=READY pgs=255 cs=0 l=1 rev1=1 rx=0x7fc974009fb0 tx=0x7fc974009b30).stop 2023-11-17T05:28:22.535 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:22.532+0000 7fc984acc700 1 -- 172.21.15.66:0/2414588887 shutdown_connections 2023-11-17T05:28:22.535 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:22.532+0000 7fc984acc700 1 --2- 172.21.15.66:0/2414588887 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fc9800f6510 0x7fc9800f6930 unknown :-1 s=CLOSED pgs=255 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:28:22.535 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:22.532+0000 7fc984acc700 1 -- 172.21.15.66:0/2414588887 >> 172.21.15.66:0/2414588887 conn(0x7fc9800f1c70 msgr2=0x7fc9800f4090 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:28:22.535 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:22.532+0000 7fc984acc700 1 -- 172.21.15.66:0/2414588887 shutdown_connections 2023-11-17T05:28:22.535 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:22.532+0000 7fc984acc700 1 -- 172.21.15.66:0/2414588887 wait complete. 2023-11-17T05:28:22.535 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:22.533+0000 7fc984acc700 1 Processor -- start 2023-11-17T05:28:22.535 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:22.533+0000 7fc984acc700 1 -- start start 2023-11-17T05:28:22.535 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:22.533+0000 7fc984acc700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fc9800f6510 0x7fc98006b9a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:28:22.535 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:22.533+0000 7fc984acc700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fc980069e20 con 0x7fc9800f6510 2023-11-17T05:28:22.535 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:22.533+0000 7fc97e59c700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fc9800f6510 0x7fc98006b9a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:28:22.536 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:22.533+0000 7fc97e59c700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fc9800f6510 0x7fc98006b9a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.66:58600/0 (socket says 172.21.15.66:58600) 2023-11-17T05:28:22.536 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:22.533+0000 7fc97e59c700 1 -- 172.21.15.66:0/3523901315 learned_addr learned my addr 172.21.15.66:0/3523901315 (peer_addr_for_me v2:172.21.15.66:0/0) 2023-11-17T05:28:22.536 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:22.533+0000 7fc97e59c700 1 -- 172.21.15.66:0/3523901315 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fc974005040 con 0x7fc9800f6510 2023-11-17T05:28:22.536 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:22.533+0000 7fc97e59c700 1 --2- 172.21.15.66:0/3523901315 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fc9800f6510 0x7fc98006b9a0 secure :-1 s=READY pgs=256 cs=0 l=1 rev1=1 rx=0x7fc974014000 tx=0x7fc97400b040).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:28:22.536 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:22.534+0000 7fc96effd700 1 -- 172.21.15.66:0/3523901315 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7fc9740095f0 con 0x7fc9800f6510 2023-11-17T05:28:22.536 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:22.534+0000 7fc984acc700 1 -- 172.21.15.66:0/3523901315 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fc980069fc0 con 0x7fc9800f6510 2023-11-17T05:28:22.536 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:22.534+0000 7fc96effd700 1 -- 172.21.15.66:0/3523901315 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(25 keys) v1 ==== 1074+0+0 (secure 0 0 0) 0x7fc974007e70 con 0x7fc9800f6510 2023-11-17T05:28:22.536 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:22.534+0000 7fc96effd700 1 -- 172.21.15.66:0/3523901315 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7fc974014ea0 con 0x7fc9800f6510 2023-11-17T05:28:22.536 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:22.534+0000 7fc984acc700 1 -- 172.21.15.66:0/3523901315 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fc98006a450 con 0x7fc9800f6510 2023-11-17T05:28:22.536 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:22.534+0000 7fc96effd700 1 -- 172.21.15.66:0/3523901315 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 17) v1 ==== 36269+0+0 (secure 0 0 0) 0x7fc974025070 con 0x7fc9800f6510 2023-11-17T05:28:22.536 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:22.534+0000 7fc984acc700 1 -- 172.21.15.66:0/3523901315 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fc980059100 con 0x7fc9800f6510 2023-11-17T05:28:22.537 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:22.535+0000 7fc96effd700 1 --2- 172.21.15.66:0/3523901315 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fc96802ff60 0x7fc968032420 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:28:22.537 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:22.535+0000 7fc96effd700 1 -- 172.21.15.66:0/3523901315 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(5..5 src has 1..5) v4 ==== 1270+0+0 (secure 0 0 0) 0x7fc974040960 con 0x7fc9800f6510 2023-11-17T05:28:22.537 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:22.535+0000 7fc97dd9b700 1 --2- 172.21.15.66:0/3523901315 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fc96802ff60 0x7fc968032420 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:28:22.537 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:22.535+0000 7fc97dd9b700 1 --2- 172.21.15.66:0/3523901315 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fc96802ff60 0x7fc968032420 secure :-1 s=READY pgs=93 cs=0 l=1 rev1=1 rx=0x7fc9700096a0 tx=0x7fc970006b40).ready entity=mgr.14164 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:28:22.540 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:22.539+0000 7fc96effd700 1 -- 172.21.15.66:0/3523901315 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+152212 (secure 0 0 0) 0x7fc9740079c0 con 0x7fc9800f6510 2023-11-17T05:28:22.760 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:22.760+0000 7fc984acc700 1 -- 172.21.15.66:0/3523901315 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "mon dump", "format": "json"} v 0) v1 -- 0x7fc9800fbf60 con 0x7fc9800f6510 2023-11-17T05:28:22.761 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:22.761+0000 7fc96effd700 1 -- 172.21.15.66:0/3523901315 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mon_command_ack([{"prefix": "mon dump", "format": "json"}]=0 dumped monmap epoch 1 v1) v1 ==== 95+0+690 (secure 0 0 0) 0x7fc97400f300 con 0x7fc9800f6510 2023-11-17T05:28:22.761 INFO:teuthology.orchestra.run.smithi066.stdout: 2023-11-17T05:28:22.761 INFO:teuthology.orchestra.run.smithi066.stdout:{"epoch":1,"fsid":"ed6470e2-8508-11ee-8dbe-212e2dc638e7","modified":"2023-11-17T05:20:13.934038Z","created":"2023-11-17T05:20:13.934038Z","min_mon_release":16,"min_mon_release_name":"pacific","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging"],"optional":[]},"mons":[{"rank":0,"name":"smithi040","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:3300","nonce":0},{"type":"v1","addr":"172.21.15.40:6789","nonce":0}]},"addr":"172.21.15.40:6789/0","public_addr":"172.21.15.40:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2023-11-17T05:28:22.762 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:22.762+0000 7fc984acc700 1 -- 172.21.15.66:0/3523901315 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fc96802ff60 msgr2=0x7fc968032420 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:28:22.762 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:22.762+0000 7fc984acc700 1 --2- 172.21.15.66:0/3523901315 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fc96802ff60 0x7fc968032420 secure :-1 s=READY pgs=93 cs=0 l=1 rev1=1 rx=0x7fc9700096a0 tx=0x7fc970006b40).stop 2023-11-17T05:28:22.762 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:22.762+0000 7fc984acc700 1 -- 172.21.15.66:0/3523901315 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fc9800f6510 msgr2=0x7fc98006b9a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:28:22.763 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:22.762+0000 7fc984acc700 1 --2- 172.21.15.66:0/3523901315 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fc9800f6510 0x7fc98006b9a0 secure :-1 s=READY pgs=256 cs=0 l=1 rev1=1 rx=0x7fc974014000 tx=0x7fc97400b040).stop 2023-11-17T05:28:22.763 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:22.762+0000 7fc984acc700 1 -- 172.21.15.66:0/3523901315 shutdown_connections 2023-11-17T05:28:22.763 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:22.762+0000 7fc984acc700 1 --2- 172.21.15.66:0/3523901315 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fc96802ff60 0x7fc968032420 unknown :-1 s=CLOSED pgs=93 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:28:22.763 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:22.762+0000 7fc984acc700 1 --2- 172.21.15.66:0/3523901315 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fc9800f6510 0x7fc98006b9a0 unknown :-1 s=CLOSED pgs=256 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:28:22.763 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:22.762+0000 7fc984acc700 1 -- 172.21.15.66:0/3523901315 >> 172.21.15.66:0/3523901315 conn(0x7fc9800f1c70 msgr2=0x7fc9800f2760 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:28:22.763 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:22.762+0000 7fc984acc700 1 -- 172.21.15.66:0/3523901315 shutdown_connections 2023-11-17T05:28:22.763 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:22.762+0000 7fc984acc700 1 -- 172.21.15.66:0/3523901315 wait complete. 2023-11-17T05:28:22.763 INFO:teuthology.orchestra.run.smithi066.stderr:dumped monmap epoch 1 2023-11-17T05:28:22.989 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:28:22 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:28:21.227139+0000 mgr.smithi040.ogcprl (mgr.14164) 221 : cluster [DBG] pgmap v209: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:28:23.989 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:28:23 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:28:22.761287+0000 mon.smithi040 (mon.0) 224 : audit [DBG] from='client.? 172.21.15.66:0/3523901315' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-11-17T05:28:24.233 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2023-11-17T05:28:24.234 DEBUG:teuthology.orchestra.run.smithi066:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.4 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ed6470e2-8508-11ee-8dbe-212e2dc638e7 -- ceph mon dump -f json 2023-11-17T05:28:24.988 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:28:24 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:28:23.227485+0000 mgr.smithi040.ogcprl (mgr.14164) 222 : cluster [DBG] pgmap v210: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:28:25.403 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:25.403+0000 7f264d5a8700 1 -- 172.21.15.66:0/4272567420 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f26480f5410 msgr2=0x7f26480f5830 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:28:25.404 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:25.403+0000 7f264d5a8700 1 --2- 172.21.15.66:0/4272567420 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f26480f5410 0x7f26480f5830 secure :-1 s=READY pgs=257 cs=0 l=1 rev1=1 rx=0x7f2638009fb0 tx=0x7f2638009b30).stop 2023-11-17T05:28:25.404 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:25.403+0000 7f264d5a8700 1 -- 172.21.15.66:0/4272567420 shutdown_connections 2023-11-17T05:28:25.404 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:25.403+0000 7f264d5a8700 1 --2- 172.21.15.66:0/4272567420 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f26480f5410 0x7f26480f5830 unknown :-1 s=CLOSED pgs=257 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:28:25.404 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:25.403+0000 7f264d5a8700 1 -- 172.21.15.66:0/4272567420 >> 172.21.15.66:0/4272567420 conn(0x7f26480f0b70 msgr2=0x7f26480f2f90 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:28:25.404 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:25.403+0000 7f264d5a8700 1 -- 172.21.15.66:0/4272567420 shutdown_connections 2023-11-17T05:28:25.404 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:25.403+0000 7f264d5a8700 1 -- 172.21.15.66:0/4272567420 wait complete. 2023-11-17T05:28:25.405 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:25.404+0000 7f264d5a8700 1 Processor -- start 2023-11-17T05:28:25.406 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:25.404+0000 7f264d5a8700 1 -- start start 2023-11-17T05:28:25.406 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:25.404+0000 7f264d5a8700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f26480f5410 0x7f26481066d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:28:25.406 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:25.404+0000 7f264d5a8700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f2648106bc0 con 0x7f26480f5410 2023-11-17T05:28:25.406 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:25.404+0000 7f2646ffd700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f26480f5410 0x7f26481066d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:28:25.406 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:25.404+0000 7f2646ffd700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f26480f5410 0x7f26481066d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.66:58618/0 (socket says 172.21.15.66:58618) 2023-11-17T05:28:25.406 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:25.404+0000 7f2646ffd700 1 -- 172.21.15.66:0/2075930705 learned_addr learned my addr 172.21.15.66:0/2075930705 (peer_addr_for_me v2:172.21.15.66:0/0) 2023-11-17T05:28:25.406 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:25.404+0000 7f2646ffd700 1 -- 172.21.15.66:0/2075930705 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f2638005040 con 0x7f26480f5410 2023-11-17T05:28:25.406 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:25.404+0000 7f2646ffd700 1 --2- 172.21.15.66:0/2075930705 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f26480f5410 0x7f26481066d0 secure :-1 s=READY pgs=258 cs=0 l=1 rev1=1 rx=0x7f2638013510 tx=0x7f263800b040).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:28:25.406 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:25.405+0000 7f263f7fe700 1 -- 172.21.15.66:0/2075930705 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f2638009320 con 0x7f26480f5410 2023-11-17T05:28:25.406 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:25.405+0000 7f264d5a8700 1 -- 172.21.15.66:0/2075930705 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f2648106dc0 con 0x7f26480f5410 2023-11-17T05:28:25.406 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:25.405+0000 7f263f7fe700 1 -- 172.21.15.66:0/2075930705 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(25 keys) v1 ==== 1074+0+0 (secure 0 0 0) 0x7f2638013f00 con 0x7f26480f5410 2023-11-17T05:28:25.406 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:25.405+0000 7f263f7fe700 1 -- 172.21.15.66:0/2075930705 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f2638014d90 con 0x7f26480f5410 2023-11-17T05:28:25.407 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:25.405+0000 7f264d5a8700 1 -- 172.21.15.66:0/2075930705 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f2648107250 con 0x7f26480f5410 2023-11-17T05:28:25.407 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:25.406+0000 7f263f7fe700 1 -- 172.21.15.66:0/2075930705 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 17) v1 ==== 36269+0+0 (secure 0 0 0) 0x7f26380095f0 con 0x7f26480f5410 2023-11-17T05:28:25.407 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:25.405+0000 7f264d5a8700 1 -- 172.21.15.66:0/2075930705 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f2648045ad0 con 0x7f26480f5410 2023-11-17T05:28:25.407 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:25.406+0000 7f263f7fe700 1 --2- 172.21.15.66:0/2075930705 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f263402ff60 0x7f2634032420 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:28:25.407 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:25.406+0000 7f263f7fe700 1 -- 172.21.15.66:0/2075930705 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(5..5 src has 1..5) v4 ==== 1270+0+0 (secure 0 0 0) 0x7f2638041000 con 0x7f26480f5410 2023-11-17T05:28:25.408 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:25.406+0000 7f26467fc700 1 --2- 172.21.15.66:0/2075930705 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f263402ff60 0x7f2634032420 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:28:25.408 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:25.406+0000 7f26467fc700 1 --2- 172.21.15.66:0/2075930705 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f263402ff60 0x7f2634032420 secure :-1 s=READY pgs=94 cs=0 l=1 rev1=1 rx=0x7f26300096a0 tx=0x7f2630006b40).ready entity=mgr.14164 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:28:25.411 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:25.411+0000 7f263f7fe700 1 -- 172.21.15.66:0/2075930705 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+152212 (secure 0 0 0) 0x7f2638014420 con 0x7f26480f5410 2023-11-17T05:28:25.631 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:25.630+0000 7f264d5a8700 1 -- 172.21.15.66:0/2075930705 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "mon dump", "format": "json"} v 0) v1 -- 0x7f26480fae60 con 0x7f26480f5410 2023-11-17T05:28:25.631 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:25.631+0000 7f263f7fe700 1 -- 172.21.15.66:0/2075930705 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mon_command_ack([{"prefix": "mon dump", "format": "json"}]=0 dumped monmap epoch 1 v1) v1 ==== 95+0+690 (secure 0 0 0) 0x7f2638010230 con 0x7f26480f5410 2023-11-17T05:28:25.632 INFO:teuthology.orchestra.run.smithi066.stdout: 2023-11-17T05:28:25.632 INFO:teuthology.orchestra.run.smithi066.stdout:{"epoch":1,"fsid":"ed6470e2-8508-11ee-8dbe-212e2dc638e7","modified":"2023-11-17T05:20:13.934038Z","created":"2023-11-17T05:20:13.934038Z","min_mon_release":16,"min_mon_release_name":"pacific","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging"],"optional":[]},"mons":[{"rank":0,"name":"smithi040","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:3300","nonce":0},{"type":"v1","addr":"172.21.15.40:6789","nonce":0}]},"addr":"172.21.15.40:6789/0","public_addr":"172.21.15.40:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2023-11-17T05:28:25.633 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:25.632+0000 7f264d5a8700 1 -- 172.21.15.66:0/2075930705 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f263402ff60 msgr2=0x7f2634032420 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:28:25.633 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:25.632+0000 7f264d5a8700 1 --2- 172.21.15.66:0/2075930705 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f263402ff60 0x7f2634032420 secure :-1 s=READY pgs=94 cs=0 l=1 rev1=1 rx=0x7f26300096a0 tx=0x7f2630006b40).stop 2023-11-17T05:28:25.633 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:25.632+0000 7f264d5a8700 1 -- 172.21.15.66:0/2075930705 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f26480f5410 msgr2=0x7f26481066d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:28:25.633 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:25.632+0000 7f264d5a8700 1 --2- 172.21.15.66:0/2075930705 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f26480f5410 0x7f26481066d0 secure :-1 s=READY pgs=258 cs=0 l=1 rev1=1 rx=0x7f2638013510 tx=0x7f263800b040).stop 2023-11-17T05:28:25.633 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:25.632+0000 7f264d5a8700 1 -- 172.21.15.66:0/2075930705 shutdown_connections 2023-11-17T05:28:25.633 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:25.632+0000 7f264d5a8700 1 --2- 172.21.15.66:0/2075930705 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f263402ff60 0x7f2634032420 unknown :-1 s=CLOSED pgs=94 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:28:25.633 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:25.632+0000 7f264d5a8700 1 --2- 172.21.15.66:0/2075930705 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f26480f5410 0x7f26481066d0 unknown :-1 s=CLOSED pgs=258 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:28:25.633 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:25.632+0000 7f264d5a8700 1 -- 172.21.15.66:0/2075930705 >> 172.21.15.66:0/2075930705 conn(0x7f26480f0b70 msgr2=0x7f264805fde0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:28:25.633 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:25.633+0000 7f264d5a8700 1 -- 172.21.15.66:0/2075930705 shutdown_connections 2023-11-17T05:28:25.633 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:25.633+0000 7f264d5a8700 1 -- 172.21.15.66:0/2075930705 wait complete. 2023-11-17T05:28:25.634 INFO:teuthology.orchestra.run.smithi066.stderr:dumped monmap epoch 1 2023-11-17T05:28:26.989 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:28:26 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:28:25.227931+0000 mgr.smithi040.ogcprl (mgr.14164) 223 : cluster [DBG] pgmap v211: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:28:26.989 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:28:26 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:28:25.631423+0000 mon.smithi040 (mon.0) 225 : audit [DBG] from='client.? 172.21.15.66:0/2075930705' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-11-17T05:28:27.006 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2023-11-17T05:28:27.006 DEBUG:teuthology.orchestra.run.smithi066:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.4 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ed6470e2-8508-11ee-8dbe-212e2dc638e7 -- ceph mon dump -f json 2023-11-17T05:28:28.194 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:28.193+0000 7f140fe1a700 1 -- 172.21.15.66:0/1663149828 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f14080f5410 msgr2=0x7f14080f5830 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:28:28.195 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:28.193+0000 7f140fe1a700 1 --2- 172.21.15.66:0/1663149828 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f14080f5410 0x7f14080f5830 secure :-1 s=READY pgs=259 cs=0 l=1 rev1=1 rx=0x7f1404009fb0 tx=0x7f1404009b30).stop 2023-11-17T05:28:28.195 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:28.194+0000 7f140fe1a700 1 -- 172.21.15.66:0/1663149828 shutdown_connections 2023-11-17T05:28:28.195 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:28.194+0000 7f140fe1a700 1 --2- 172.21.15.66:0/1663149828 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f14080f5410 0x7f14080f5830 unknown :-1 s=CLOSED pgs=259 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:28:28.195 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:28.194+0000 7f140fe1a700 1 -- 172.21.15.66:0/1663149828 >> 172.21.15.66:0/1663149828 conn(0x7f14080f0b70 msgr2=0x7f14080f2f90 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:28:28.195 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:28.194+0000 7f140fe1a700 1 -- 172.21.15.66:0/1663149828 shutdown_connections 2023-11-17T05:28:28.195 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:28.194+0000 7f140fe1a700 1 -- 172.21.15.66:0/1663149828 wait complete. 2023-11-17T05:28:28.195 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:28.194+0000 7f140fe1a700 1 Processor -- start 2023-11-17T05:28:28.195 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:28.194+0000 7f140fe1a700 1 -- start start 2023-11-17T05:28:28.195 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:28.194+0000 7f140fe1a700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f14080f5410 0x7f1408109450 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:28:28.195 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:28.194+0000 7f140fe1a700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f1408060220 con 0x7f14080f5410 2023-11-17T05:28:28.197 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:28.195+0000 7f140dbb6700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f14080f5410 0x7f1408109450 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:28:28.197 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:28.195+0000 7f140dbb6700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f14080f5410 0x7f1408109450 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.66:58640/0 (socket says 172.21.15.66:58640) 2023-11-17T05:28:28.197 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:28.195+0000 7f140dbb6700 1 -- 172.21.15.66:0/190054178 learned_addr learned my addr 172.21.15.66:0/190054178 (peer_addr_for_me v2:172.21.15.66:0/0) 2023-11-17T05:28:28.197 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:28.195+0000 7f140dbb6700 1 -- 172.21.15.66:0/190054178 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f1404005040 con 0x7f14080f5410 2023-11-17T05:28:28.197 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:28.195+0000 7f140dbb6700 1 --2- 172.21.15.66:0/190054178 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f14080f5410 0x7f1408109450 secure :-1 s=READY pgs=260 cs=0 l=1 rev1=1 rx=0x7f1404013510 tx=0x7f140400b040).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:28:28.197 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:28.195+0000 7f13fa7fc700 1 -- 172.21.15.66:0/190054178 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f1404009320 con 0x7f14080f5410 2023-11-17T05:28:28.197 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:28.195+0000 7f140fe1a700 1 -- 172.21.15.66:0/190054178 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f1408060420 con 0x7f14080f5410 2023-11-17T05:28:28.197 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:28.195+0000 7f13fa7fc700 1 -- 172.21.15.66:0/190054178 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(25 keys) v1 ==== 1074+0+0 (secure 0 0 0) 0x7f1404013f00 con 0x7f14080f5410 2023-11-17T05:28:28.198 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:28.195+0000 7f13fa7fc700 1 -- 172.21.15.66:0/190054178 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f1404014d90 con 0x7f14080f5410 2023-11-17T05:28:28.198 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:28.195+0000 7f140fe1a700 1 -- 172.21.15.66:0/190054178 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f1408109b00 con 0x7f14080f5410 2023-11-17T05:28:28.198 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:28.196+0000 7f13fa7fc700 1 -- 172.21.15.66:0/190054178 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 17) v1 ==== 36269+0+0 (secure 0 0 0) 0x7f14040095f0 con 0x7f14080f5410 2023-11-17T05:28:28.198 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:28.196+0000 7f140fe1a700 1 -- 172.21.15.66:0/190054178 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f1408059100 con 0x7f14080f5410 2023-11-17T05:28:28.198 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:28.196+0000 7f13fa7fc700 1 --2- 172.21.15.66:0/190054178 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f13f402ff60 0x7f13f4032420 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:28:28.198 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:28.196+0000 7f13fa7fc700 1 -- 172.21.15.66:0/190054178 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(5..5 src has 1..5) v4 ==== 1270+0+0 (secure 0 0 0) 0x7f1404040fe0 con 0x7f14080f5410 2023-11-17T05:28:28.198 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:28.197+0000 7f140d3b5700 1 --2- 172.21.15.66:0/190054178 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f13f402ff60 0x7f13f4032420 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:28:28.198 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:28.197+0000 7f140d3b5700 1 --2- 172.21.15.66:0/190054178 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f13f402ff60 0x7f13f4032420 secure :-1 s=READY pgs=95 cs=0 l=1 rev1=1 rx=0x7f13fc0096a0 tx=0x7f13fc006b40).ready entity=mgr.14164 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:28:28.201 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:28.201+0000 7f13fa7fc700 1 -- 172.21.15.66:0/190054178 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+152212 (secure 0 0 0) 0x7f140401d7f0 con 0x7f14080f5410 2023-11-17T05:28:28.422 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:28.421+0000 7f140fe1a700 1 -- 172.21.15.66:0/190054178 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "mon dump", "format": "json"} v 0) v1 -- 0x7f1408045ad0 con 0x7f14080f5410 2023-11-17T05:28:28.422 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:28.422+0000 7f13fa7fc700 1 -- 172.21.15.66:0/190054178 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mon_command_ack([{"prefix": "mon dump", "format": "json"}]=0 dumped monmap epoch 1 v1) v1 ==== 95+0+690 (secure 0 0 0) 0x7f1404010230 con 0x7f14080f5410 2023-11-17T05:28:28.423 INFO:teuthology.orchestra.run.smithi066.stdout: 2023-11-17T05:28:28.423 INFO:teuthology.orchestra.run.smithi066.stdout:{"epoch":1,"fsid":"ed6470e2-8508-11ee-8dbe-212e2dc638e7","modified":"2023-11-17T05:20:13.934038Z","created":"2023-11-17T05:20:13.934038Z","min_mon_release":16,"min_mon_release_name":"pacific","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging"],"optional":[]},"mons":[{"rank":0,"name":"smithi040","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:3300","nonce":0},{"type":"v1","addr":"172.21.15.40:6789","nonce":0}]},"addr":"172.21.15.40:6789/0","public_addr":"172.21.15.40:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2023-11-17T05:28:28.424 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:28.424+0000 7f140fe1a700 1 -- 172.21.15.66:0/190054178 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f13f402ff60 msgr2=0x7f13f4032420 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:28:28.424 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:28.424+0000 7f140fe1a700 1 --2- 172.21.15.66:0/190054178 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f13f402ff60 0x7f13f4032420 secure :-1 s=READY pgs=95 cs=0 l=1 rev1=1 rx=0x7f13fc0096a0 tx=0x7f13fc006b40).stop 2023-11-17T05:28:28.424 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:28.424+0000 7f140fe1a700 1 -- 172.21.15.66:0/190054178 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f14080f5410 msgr2=0x7f1408109450 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:28:28.424 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:28.424+0000 7f140fe1a700 1 --2- 172.21.15.66:0/190054178 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f14080f5410 0x7f1408109450 secure :-1 s=READY pgs=260 cs=0 l=1 rev1=1 rx=0x7f1404013510 tx=0x7f140400b040).stop 2023-11-17T05:28:28.424 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:28.424+0000 7f140fe1a700 1 -- 172.21.15.66:0/190054178 shutdown_connections 2023-11-17T05:28:28.425 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:28.424+0000 7f140fe1a700 1 --2- 172.21.15.66:0/190054178 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f13f402ff60 0x7f13f4032420 unknown :-1 s=CLOSED pgs=95 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:28:28.425 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:28.424+0000 7f140fe1a700 1 --2- 172.21.15.66:0/190054178 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f14080f5410 0x7f1408109450 unknown :-1 s=CLOSED pgs=260 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:28:28.425 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:28.424+0000 7f140fe1a700 1 -- 172.21.15.66:0/190054178 >> 172.21.15.66:0/190054178 conn(0x7f14080f0b70 msgr2=0x7f14080f1660 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:28:28.425 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:28.424+0000 7f140fe1a700 1 -- 172.21.15.66:0/190054178 shutdown_connections 2023-11-17T05:28:28.425 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:28.424+0000 7f140fe1a700 1 -- 172.21.15.66:0/190054178 wait complete. 2023-11-17T05:28:28.425 INFO:teuthology.orchestra.run.smithi066.stderr:dumped monmap epoch 1 2023-11-17T05:28:28.989 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:28:28 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:28:27.228299+0000 mgr.smithi040.ogcprl (mgr.14164) 224 : cluster [DBG] pgmap v212: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:28:28.989 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:28:28 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:28:28.422761+0000 mon.smithi040 (mon.0) 226 : audit [DBG] from='client.? 172.21.15.66:0/190054178' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-11-17T05:28:29.908 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2023-11-17T05:28:29.908 DEBUG:teuthology.orchestra.run.smithi066:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.4 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ed6470e2-8508-11ee-8dbe-212e2dc638e7 -- ceph mon dump -f json 2023-11-17T05:28:30.739 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:28:30 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:28:29.228661+0000 mgr.smithi040.ogcprl (mgr.14164) 225 : cluster [DBG] pgmap v213: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:28:31.143 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:31.141+0000 7f492b84c700 1 -- 172.21.15.66:0/507045638 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f49240601e0 msgr2=0x7f49240f9190 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:28:31.143 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:31.141+0000 7f492b84c700 1 --2- 172.21.15.66:0/507045638 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f49240601e0 0x7f49240f9190 secure :-1 s=READY pgs=261 cs=0 l=1 rev1=1 rx=0x7f4920009fb0 tx=0x7f4920009b30).stop 2023-11-17T05:28:31.143 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:31.141+0000 7f492b84c700 1 -- 172.21.15.66:0/507045638 shutdown_connections 2023-11-17T05:28:31.143 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:31.141+0000 7f492b84c700 1 --2- 172.21.15.66:0/507045638 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f49240601e0 0x7f49240f9190 unknown :-1 s=CLOSED pgs=261 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:28:31.143 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:31.141+0000 7f492b84c700 1 -- 172.21.15.66:0/507045638 >> 172.21.15.66:0/507045638 conn(0x7f49240f0b70 msgr2=0x7f49240f2f90 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:28:31.143 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:31.141+0000 7f492b84c700 1 -- 172.21.15.66:0/507045638 shutdown_connections 2023-11-17T05:28:31.143 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:31.141+0000 7f492b84c700 1 -- 172.21.15.66:0/507045638 wait complete. 2023-11-17T05:28:31.143 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:31.142+0000 7f492b84c700 1 Processor -- start 2023-11-17T05:28:31.143 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:31.142+0000 7f492b84c700 1 -- start start 2023-11-17T05:28:31.144 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:31.142+0000 7f492b84c700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f49240601e0 0x7f49241066e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:28:31.144 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:31.142+0000 7f492b84c700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f4924106bd0 con 0x7f49240601e0 2023-11-17T05:28:31.144 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:31.143+0000 7f49295e8700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f49240601e0 0x7f49241066e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:28:31.144 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:31.143+0000 7f49295e8700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f49240601e0 0x7f49241066e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.66:58652/0 (socket says 172.21.15.66:58652) 2023-11-17T05:28:31.144 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:31.143+0000 7f49295e8700 1 -- 172.21.15.66:0/1816141948 learned_addr learned my addr 172.21.15.66:0/1816141948 (peer_addr_for_me v2:172.21.15.66:0/0) 2023-11-17T05:28:31.144 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:31.143+0000 7f49295e8700 1 -- 172.21.15.66:0/1816141948 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f4920005040 con 0x7f49240601e0 2023-11-17T05:28:31.145 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:31.143+0000 7f49295e8700 1 --2- 172.21.15.66:0/1816141948 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f49240601e0 0x7f49241066e0 secure :-1 s=READY pgs=262 cs=0 l=1 rev1=1 rx=0x7f4920004dd0 tx=0x7f4920009b30).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:28:31.145 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:31.143+0000 7f4919ffb700 1 -- 172.21.15.66:0/1816141948 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f4920018070 con 0x7f49240601e0 2023-11-17T05:28:31.145 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:31.143+0000 7f492b84c700 1 -- 172.21.15.66:0/1816141948 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f4924106dd0 con 0x7f49240601e0 2023-11-17T05:28:31.145 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:31.143+0000 7f4919ffb700 1 -- 172.21.15.66:0/1816141948 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(25 keys) v1 ==== 1074+0+0 (secure 0 0 0) 0x7f4920013410 con 0x7f49240601e0 2023-11-17T05:28:31.146 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:31.143+0000 7f4919ffb700 1 -- 172.21.15.66:0/1816141948 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f492001c3c0 con 0x7f49240601e0 2023-11-17T05:28:31.146 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:31.143+0000 7f492b84c700 1 -- 172.21.15.66:0/1816141948 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f4924107260 con 0x7f49240601e0 2023-11-17T05:28:31.146 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:31.144+0000 7f4919ffb700 1 -- 172.21.15.66:0/1816141948 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 17) v1 ==== 36269+0+0 (secure 0 0 0) 0x7f492001d420 con 0x7f49240601e0 2023-11-17T05:28:31.146 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:31.144+0000 7f492b84c700 1 -- 172.21.15.66:0/1816141948 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f4924045ad0 con 0x7f49240601e0 2023-11-17T05:28:31.146 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:31.144+0000 7f4919ffb700 1 --2- 172.21.15.66:0/1816141948 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f491002ff60 0x7f4910032420 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:28:31.146 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:31.144+0000 7f4919ffb700 1 -- 172.21.15.66:0/1816141948 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(5..5 src has 1..5) v4 ==== 1270+0+0 (secure 0 0 0) 0x7f4920041390 con 0x7f49240601e0 2023-11-17T05:28:31.146 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:31.145+0000 7f4928de7700 1 --2- 172.21.15.66:0/1816141948 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f491002ff60 0x7f4910032420 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:28:31.146 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:31.145+0000 7f4928de7700 1 --2- 172.21.15.66:0/1816141948 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f491002ff60 0x7f4910032420 secure :-1 s=READY pgs=96 cs=0 l=1 rev1=1 rx=0x7f49140096a0 tx=0x7f4914006b40).ready entity=mgr.14164 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:28:31.149 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:31.149+0000 7f4919ffb700 1 -- 172.21.15.66:0/1816141948 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+152212 (secure 0 0 0) 0x7f492001d820 con 0x7f49240601e0 2023-11-17T05:28:31.368 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:31.367+0000 7f492b84c700 1 -- 172.21.15.66:0/1816141948 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "mon dump", "format": "json"} v 0) v1 -- 0x7f49241080b0 con 0x7f49240601e0 2023-11-17T05:28:31.368 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:31.368+0000 7f4919ffb700 1 -- 172.21.15.66:0/1816141948 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mon_command_ack([{"prefix": "mon dump", "format": "json"}]=0 dumped monmap epoch 1 v1) v1 ==== 95+0+690 (secure 0 0 0) 0x7f492001b370 con 0x7f49240601e0 2023-11-17T05:28:31.368 INFO:teuthology.orchestra.run.smithi066.stdout: 2023-11-17T05:28:31.368 INFO:teuthology.orchestra.run.smithi066.stdout:{"epoch":1,"fsid":"ed6470e2-8508-11ee-8dbe-212e2dc638e7","modified":"2023-11-17T05:20:13.934038Z","created":"2023-11-17T05:20:13.934038Z","min_mon_release":16,"min_mon_release_name":"pacific","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging"],"optional":[]},"mons":[{"rank":0,"name":"smithi040","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:3300","nonce":0},{"type":"v1","addr":"172.21.15.40:6789","nonce":0}]},"addr":"172.21.15.40:6789/0","public_addr":"172.21.15.40:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2023-11-17T05:28:31.369 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:31.369+0000 7f492b84c700 1 -- 172.21.15.66:0/1816141948 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f491002ff60 msgr2=0x7f4910032420 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:28:31.369 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:31.369+0000 7f492b84c700 1 --2- 172.21.15.66:0/1816141948 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f491002ff60 0x7f4910032420 secure :-1 s=READY pgs=96 cs=0 l=1 rev1=1 rx=0x7f49140096a0 tx=0x7f4914006b40).stop 2023-11-17T05:28:31.370 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:31.369+0000 7f492b84c700 1 -- 172.21.15.66:0/1816141948 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f49240601e0 msgr2=0x7f49241066e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:28:31.370 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:31.369+0000 7f492b84c700 1 --2- 172.21.15.66:0/1816141948 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f49240601e0 0x7f49241066e0 secure :-1 s=READY pgs=262 cs=0 l=1 rev1=1 rx=0x7f4920004dd0 tx=0x7f4920009b30).stop 2023-11-17T05:28:31.370 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:31.369+0000 7f492b84c700 1 -- 172.21.15.66:0/1816141948 shutdown_connections 2023-11-17T05:28:31.370 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:31.369+0000 7f492b84c700 1 --2- 172.21.15.66:0/1816141948 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f491002ff60 0x7f4910032420 unknown :-1 s=CLOSED pgs=96 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:28:31.370 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:31.369+0000 7f492b84c700 1 --2- 172.21.15.66:0/1816141948 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f49240601e0 0x7f49241066e0 unknown :-1 s=CLOSED pgs=262 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:28:31.370 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:31.369+0000 7f492b84c700 1 -- 172.21.15.66:0/1816141948 >> 172.21.15.66:0/1816141948 conn(0x7f49240f0b70 msgr2=0x7f49240f1670 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:28:31.370 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:31.369+0000 7f492b84c700 1 -- 172.21.15.66:0/1816141948 shutdown_connections 2023-11-17T05:28:31.370 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:31.369+0000 7f492b84c700 1 -- 172.21.15.66:0/1816141948 wait complete. 2023-11-17T05:28:31.370 INFO:teuthology.orchestra.run.smithi066.stderr:dumped monmap epoch 1 2023-11-17T05:28:31.739 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:28:31 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:28:31.368313+0000 mon.smithi040 (mon.0) 227 : audit [DBG] from='client.? 172.21.15.66:0/1816141948' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-11-17T05:28:32.739 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:28:32 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:28:31.229030+0000 mgr.smithi040.ogcprl (mgr.14164) 226 : cluster [DBG] pgmap v214: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:28:46.652 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2023-11-17T05:28:46.653 DEBUG:teuthology.orchestra.run.smithi066:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.4 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ed6470e2-8508-11ee-8dbe-212e2dc638e7 -- ceph mon dump -f json 2023-11-17T05:28:46.656 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:28:34 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:28:33.229392+0000 mgr.smithi040.ogcprl (mgr.14164) 227 : cluster [DBG] pgmap v215: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:28:46.656 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:28:36 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:28:35.229695+0000 mgr.smithi040.ogcprl (mgr.14164) 228 : cluster [DBG] pgmap v216: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:28:46.656 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:28:38 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:28:37.230086+0000 mgr.smithi040.ogcprl (mgr.14164) 229 : cluster [DBG] pgmap v217: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:28:46.656 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:28:40 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:28:39.230429+0000 mgr.smithi040.ogcprl (mgr.14164) 230 : cluster [DBG] pgmap v218: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:28:46.656 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:28:42 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cephadm 2023-11-17T05:28:40.531152+0000 mgr.smithi040.ogcprl (mgr.14164) 231 : cephadm [ERR] cephadm exited with an error code: 1, stderr:Deploy daemon node-exporter.smithi040 ... 2023-11-17T05:28:46.656 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:28:42 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: Verifying port 9100 ... 2023-11-17T05:28:46.656 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:28:42 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: Non-zero exit code 1 from systemctl start ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7@node-exporter.smithi040 2023-11-17T05:28:46.656 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:28:42 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: systemctl: stderr Job for ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7@node-exporter.smithi040.service failed because a timeout was exceeded. 2023-11-17T05:28:46.656 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:28:42 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: systemctl: stderr See "systemctl status ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7@node-exporter.smithi040.service" and "journalctl -xe" for details. 2023-11-17T05:28:46.656 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:28:42 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: Traceback (most recent call last): 2023-11-17T05:28:46.656 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:28:42 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: File "/var/lib/ceph/ed6470e2-8508-11ee-8dbe-212e2dc638e7/cephadm.30cb78bdbbafb384af862e1c2292b944f15942b586128e91262b43e91e11ae90", line 8029, in 2023-11-17T05:28:46.657 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:28:42 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: main() 2023-11-17T05:28:46.657 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:28:42 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: File "/var/lib/ceph/ed6470e2-8508-11ee-8dbe-212e2dc638e7/cephadm.30cb78bdbbafb384af862e1c2292b944f15942b586128e91262b43e91e11ae90", line 8017, in main 2023-11-17T05:28:46.657 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:28:42 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: r = ctx.func(ctx) 2023-11-17T05:28:46.657 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:28:42 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: File "/var/lib/ceph/ed6470e2-8508-11ee-8dbe-212e2dc638e7/cephadm.30cb78bdbbafb384af862e1c2292b944f15942b586128e91262b43e91e11ae90", line 1760, in _default_image 2023-11-17T05:28:46.657 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:28:42 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: return func(ctx) 2023-11-17T05:28:46.657 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:28:42 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: File "/var/lib/ceph/ed6470e2-8508-11ee-8dbe-212e2dc638e7/cephadm.30cb78bdbbafb384af862e1c2292b944f15942b586128e91262b43e91e11ae90", line 4267, in command_deploy 2023-11-17T05:28:46.657 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:28:42 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: ports=daemon_ports) 2023-11-17T05:28:46.657 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:28:42 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: File "/var/lib/ceph/ed6470e2-8508-11ee-8dbe-212e2dc638e7/cephadm.30cb78bdbbafb384af862e1c2292b944f15942b586128e91262b43e91e11ae90", line 2608, in deploy_daemon 2023-11-17T05:28:46.657 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:28:42 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: c, osd_fsid=osd_fsid, ports=ports) 2023-11-17T05:28:46.657 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:28:42 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: File "/var/lib/ceph/ed6470e2-8508-11ee-8dbe-212e2dc638e7/cephadm.30cb78bdbbafb384af862e1c2292b944f15942b586128e91262b43e91e11ae90", line 2806, in deploy_daemon_units 2023-11-17T05:28:46.657 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:28:42 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: call_throws(ctx, ['systemctl', 'start', unit_name]) 2023-11-17T05:28:46.657 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:28:42 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: File "/var/lib/ceph/ed6470e2-8508-11ee-8dbe-212e2dc638e7/cephadm.30cb78bdbbafb384af862e1c2292b944f15942b586128e91262b43e91e11ae90", line 1464, in call_throws 2023-11-17T05:28:46.657 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:28:42 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: raise RuntimeError('Failed command: %s' % ' '.join(command)) 2023-11-17T05:28:46.658 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:28:42 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: RuntimeError: Failed command: systemctl start ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7@node-exporter.smithi040 2023-11-17T05:28:46.658 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:28:42 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: Traceback (most recent call last): 2023-11-17T05:28:46.658 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:28:42 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1144, in _remote_connection 2023-11-17T05:28:46.658 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:28:42 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: yield (conn, connr) 2023-11-17T05:28:46.658 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:28:42 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1059, in _run_cephadm 2023-11-17T05:28:46.658 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:28:42 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: code, '\n'.join(err))) 2023-11-17T05:28:46.658 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:28:42 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr:Deploy daemon node-exporter.smithi040 ... 2023-11-17T05:28:46.658 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:28:42 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: Verifying port 9100 ... 2023-11-17T05:28:46.658 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:28:42 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: Non-zero exit code 1 from systemctl start ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7@node-exporter.smithi040 2023-11-17T05:28:46.658 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:28:42 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: systemctl: stderr Job for ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7@node-exporter.smithi040.service failed because a timeout was exceeded. 2023-11-17T05:28:46.658 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:28:42 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: systemctl: stderr See "systemctl status ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7@node-exporter.smithi040.service" and "journalctl -xe" for details. 2023-11-17T05:28:46.658 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:28:42 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: Traceback (most recent call last): 2023-11-17T05:28:46.659 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:28:42 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: File "/var/lib/ceph/ed6470e2-8508-11ee-8dbe-212e2dc638e7/cephadm.30cb78bdbbafb384af862e1c2292b944f15942b586128e91262b43e91e11ae90", line 8029, in 2023-11-17T05:28:46.659 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:28:42 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: main() 2023-11-17T05:28:46.659 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:28:42 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: File "/var/lib/ceph/ed6470e2-8508-11ee-8dbe-212e2dc638e7/cephadm.30cb78bdbbafb384af862e1c2292b944f15942b586128e91262b43e91e11ae90", line 8017, in main 2023-11-17T05:28:46.659 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:28:42 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: r = ctx.func(ctx) 2023-11-17T05:28:46.659 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:28:42 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: File "/var/lib/ceph/ed6470e2-8508-11ee-8dbe-212e2dc638e7/cephadm.30cb78bdbbafb384af862e1c2292b944f15942b586128e91262b43e91e11ae90", line 1760, in _default_image 2023-11-17T05:28:46.659 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:28:42 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: return func(ctx) 2023-11-17T05:28:46.659 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:28:42 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: File "/var/lib/ceph/ed6470e2-8508-11ee-8dbe-212e2dc638e7/cephadm.30cb78bdbbafb384af862e1c2292b944f15942b586128e91262b43e91e11ae90", line 4267, in command_deploy 2023-11-17T05:28:46.659 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:28:42 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: ports=daemon_ports) 2023-11-17T05:28:46.659 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:28:42 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: File "/var/lib/ceph/ed6470e2-8508-11ee-8dbe-212e2dc638e7/cephadm.30cb78bdbbafb384af862e1c2292b944f15942b586128e91262b43e91e11ae90", line 2608, in deploy_daemon 2023-11-17T05:28:46.659 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:28:42 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: c, osd_fsid=osd_fsid, ports=ports) 2023-11-17T05:28:46.659 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:28:42 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: File "/var/lib/ceph/ed6470e2-8508-11ee-8dbe-212e2dc638e7/cephadm.30cb78bdbbafb384af862e1c2292b944f15942b586128e91262b43e91e11ae90", line 2806, in deploy_daemon_units 2023-11-17T05:28:46.659 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:28:42 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: call_throws(ctx, ['systemctl', 'start', unit_name]) 2023-11-17T05:28:46.659 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:28:42 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: File "/var/lib/ceph/ed6470e2-8508-11ee-8dbe-212e2dc638e7/cephadm.30cb78bdbbafb384af862e1c2292b944f15942b586128e91262b43e91e11ae90", line 1464, in call_throws 2023-11-17T05:28:46.660 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:28:42 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: raise RuntimeError('Failed command: %s' % ' '.join(command)) 2023-11-17T05:28:46.660 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:28:42 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: RuntimeError: Failed command: systemctl start ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7@node-exporter.smithi040 2023-11-17T05:28:46.660 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:28:42 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cephadm 2023-11-17T05:28:40.537163+0000 mgr.smithi040.ogcprl (mgr.14164) 232 : cephadm [INF] Deploying daemon prometheus.smithi040 on smithi040 2023-11-17T05:28:46.660 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:28:42 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:28:41.230808+0000 mgr.smithi040.ogcprl (mgr.14164) 233 : cluster [DBG] pgmap v219: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:28:46.660 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:28:44 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:28:43.231206+0000 mgr.smithi040.ogcprl (mgr.14164) 234 : cluster [DBG] pgmap v220: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:28:46.739 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:28:46 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:28:45.231529+0000 mgr.smithi040.ogcprl (mgr.14164) 235 : cluster [DBG] pgmap v221: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:28:47.859 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:47.859+0000 7f3abf2bd700 1 -- 172.21.15.66:0/139595565 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f3ab80f51a0 msgr2=0x7f3ab80f55c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:28:47.859 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:47.859+0000 7f3abf2bd700 1 --2- 172.21.15.66:0/139595565 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f3ab80f51a0 0x7f3ab80f55c0 secure :-1 s=READY pgs=263 cs=0 l=1 rev1=1 rx=0x7f3aac009fb0 tx=0x7f3aac009b30).stop 2023-11-17T05:28:47.859 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:47.859+0000 7f3abf2bd700 1 -- 172.21.15.66:0/139595565 shutdown_connections 2023-11-17T05:28:47.860 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:47.859+0000 7f3abf2bd700 1 --2- 172.21.15.66:0/139595565 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f3ab80f51a0 0x7f3ab80f55c0 unknown :-1 s=CLOSED pgs=263 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:28:47.860 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:47.859+0000 7f3abf2bd700 1 -- 172.21.15.66:0/139595565 >> 172.21.15.66:0/139595565 conn(0x7f3ab80f09a0 msgr2=0x7f3ab80f2dc0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:28:47.860 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:47.859+0000 7f3abf2bd700 1 -- 172.21.15.66:0/139595565 shutdown_connections 2023-11-17T05:28:47.860 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:47.859+0000 7f3abf2bd700 1 -- 172.21.15.66:0/139595565 wait complete. 2023-11-17T05:28:47.861 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:47.859+0000 7f3abf2bd700 1 Processor -- start 2023-11-17T05:28:47.861 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:47.859+0000 7f3abf2bd700 1 -- start start 2023-11-17T05:28:47.861 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:47.860+0000 7f3abf2bd700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f3ab80f51a0 0x7f3ab810b5d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:28:47.861 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:47.860+0000 7f3abf2bd700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f3ab8106fc0 con 0x7f3ab80f51a0 2023-11-17T05:28:47.861 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:47.860+0000 7f3abe2bb700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f3ab80f51a0 0x7f3ab810b5d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:28:47.862 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:47.860+0000 7f3abe2bb700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f3ab80f51a0 0x7f3ab810b5d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.66:51812/0 (socket says 172.21.15.66:51812) 2023-11-17T05:28:47.862 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:47.860+0000 7f3abe2bb700 1 -- 172.21.15.66:0/1275748770 learned_addr learned my addr 172.21.15.66:0/1275748770 (peer_addr_for_me v2:172.21.15.66:0/0) 2023-11-17T05:28:47.862 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:47.860+0000 7f3abe2bb700 1 -- 172.21.15.66:0/1275748770 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f3aac005040 con 0x7f3ab80f51a0 2023-11-17T05:28:47.862 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:47.860+0000 7f3abe2bb700 1 --2- 172.21.15.66:0/1275748770 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f3ab80f51a0 0x7f3ab810b5d0 secure :-1 s=READY pgs=264 cs=0 l=1 rev1=1 rx=0x7f3aac013510 tx=0x7f3aac00b040).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:28:47.862 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:47.860+0000 7f3aaaffd700 1 -- 172.21.15.66:0/1275748770 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f3aac009320 con 0x7f3ab80f51a0 2023-11-17T05:28:47.862 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:47.860+0000 7f3aaaffd700 1 -- 172.21.15.66:0/1275748770 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(25 keys) v1 ==== 1074+0+0 (secure 0 0 0) 0x7f3aac013f00 con 0x7f3ab80f51a0 2023-11-17T05:28:47.862 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:47.860+0000 7f3abf2bd700 1 -- 172.21.15.66:0/1275748770 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f3ab8060220 con 0x7f3ab80f51a0 2023-11-17T05:28:47.862 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:47.860+0000 7f3aaaffd700 1 -- 172.21.15.66:0/1275748770 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f3aac014dc0 con 0x7f3ab80f51a0 2023-11-17T05:28:47.862 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:47.861+0000 7f3abf2bd700 1 -- 172.21.15.66:0/1275748770 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f3ab810bac0 con 0x7f3ab80f51a0 2023-11-17T05:28:47.863 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:47.861+0000 7f3aaaffd700 1 -- 172.21.15.66:0/1275748770 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 17) v1 ==== 36269+0+0 (secure 0 0 0) 0x7f3aac0095f0 con 0x7f3ab80f51a0 2023-11-17T05:28:47.863 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:47.861+0000 7f3abf2bd700 1 -- 172.21.15.66:0/1275748770 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f3ab8045ad0 con 0x7f3ab80f51a0 2023-11-17T05:28:47.863 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:47.862+0000 7f3aaaffd700 1 --2- 172.21.15.66:0/1275748770 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f3aa402ff60 0x7f3aa4032420 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:28:47.863 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:47.862+0000 7f3aaaffd700 1 -- 172.21.15.66:0/1275748770 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(5..5 src has 1..5) v4 ==== 1270+0+0 (secure 0 0 0) 0x7f3aac041040 con 0x7f3ab80f51a0 2023-11-17T05:28:47.863 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:47.862+0000 7f3abdaba700 1 --2- 172.21.15.66:0/1275748770 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f3aa402ff60 0x7f3aa4032420 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:28:47.863 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:47.862+0000 7f3abdaba700 1 --2- 172.21.15.66:0/1275748770 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f3aa402ff60 0x7f3aa4032420 secure :-1 s=READY pgs=97 cs=0 l=1 rev1=1 rx=0x7f3ab40096a0 tx=0x7f3ab4006b40).ready entity=mgr.14164 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:28:47.866 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:47.866+0000 7f3aaaffd700 1 -- 172.21.15.66:0/1275748770 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+152212 (secure 0 0 0) 0x7f3aac014420 con 0x7f3ab80f51a0 2023-11-17T05:28:48.088 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:48.088+0000 7f3abf2bd700 1 -- 172.21.15.66:0/1275748770 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "mon dump", "format": "json"} v 0) v1 -- 0x7f3ab810c280 con 0x7f3ab80f51a0 2023-11-17T05:28:48.088 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:48.088+0000 7f3aaaffd700 1 -- 172.21.15.66:0/1275748770 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mon_command_ack([{"prefix": "mon dump", "format": "json"}]=0 dumped monmap epoch 1 v1) v1 ==== 95+0+690 (secure 0 0 0) 0x7f3aac010230 con 0x7f3ab80f51a0 2023-11-17T05:28:48.089 INFO:teuthology.orchestra.run.smithi066.stdout: 2023-11-17T05:28:48.089 INFO:teuthology.orchestra.run.smithi066.stdout:{"epoch":1,"fsid":"ed6470e2-8508-11ee-8dbe-212e2dc638e7","modified":"2023-11-17T05:20:13.934038Z","created":"2023-11-17T05:20:13.934038Z","min_mon_release":16,"min_mon_release_name":"pacific","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging"],"optional":[]},"mons":[{"rank":0,"name":"smithi040","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:3300","nonce":0},{"type":"v1","addr":"172.21.15.40:6789","nonce":0}]},"addr":"172.21.15.40:6789/0","public_addr":"172.21.15.40:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2023-11-17T05:28:48.090 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:48.090+0000 7f3abf2bd700 1 -- 172.21.15.66:0/1275748770 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f3aa402ff60 msgr2=0x7f3aa4032420 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:28:48.090 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:48.090+0000 7f3abf2bd700 1 --2- 172.21.15.66:0/1275748770 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f3aa402ff60 0x7f3aa4032420 secure :-1 s=READY pgs=97 cs=0 l=1 rev1=1 rx=0x7f3ab40096a0 tx=0x7f3ab4006b40).stop 2023-11-17T05:28:48.090 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:48.090+0000 7f3abf2bd700 1 -- 172.21.15.66:0/1275748770 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f3ab80f51a0 msgr2=0x7f3ab810b5d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:28:48.090 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:48.090+0000 7f3abf2bd700 1 --2- 172.21.15.66:0/1275748770 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f3ab80f51a0 0x7f3ab810b5d0 secure :-1 s=READY pgs=264 cs=0 l=1 rev1=1 rx=0x7f3aac013510 tx=0x7f3aac00b040).stop 2023-11-17T05:28:48.091 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:48.090+0000 7f3abf2bd700 1 -- 172.21.15.66:0/1275748770 shutdown_connections 2023-11-17T05:28:48.091 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:48.090+0000 7f3abf2bd700 1 --2- 172.21.15.66:0/1275748770 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f3aa402ff60 0x7f3aa4032420 unknown :-1 s=CLOSED pgs=97 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:28:48.091 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:48.090+0000 7f3abf2bd700 1 --2- 172.21.15.66:0/1275748770 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f3ab80f51a0 0x7f3ab810b5d0 unknown :-1 s=CLOSED pgs=264 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:28:48.091 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:48.090+0000 7f3abf2bd700 1 -- 172.21.15.66:0/1275748770 >> 172.21.15.66:0/1275748770 conn(0x7f3ab80f09a0 msgr2=0x7f3ab80f1460 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:28:48.091 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:48.090+0000 7f3abf2bd700 1 -- 172.21.15.66:0/1275748770 shutdown_connections 2023-11-17T05:28:48.091 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:48.090+0000 7f3abf2bd700 1 -- 172.21.15.66:0/1275748770 wait complete. 2023-11-17T05:28:48.091 INFO:teuthology.orchestra.run.smithi066.stderr:dumped monmap epoch 1 2023-11-17T05:28:48.739 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:28:48 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:28:47.231861+0000 mgr.smithi040.ogcprl (mgr.14164) 236 : cluster [DBG] pgmap v222: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:28:53.942 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:28:48 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:28:48.088903+0000 mon.smithi040 (mon.0) 228 : audit [DBG] from='client.? 172.21.15.66:0/1275748770' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-11-17T05:28:53.942 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2023-11-17T05:28:53.943 DEBUG:teuthology.orchestra.run.smithi066:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.4 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ed6470e2-8508-11ee-8dbe-212e2dc638e7 -- ceph mon dump -f json 2023-11-17T05:28:53.944 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:28:50 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:28:49.232155+0000 mgr.smithi040.ogcprl (mgr.14164) 237 : cluster [DBG] pgmap v223: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:28:53.944 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:28:52 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:28:51.232436+0000 mgr.smithi040.ogcprl (mgr.14164) 238 : cluster [DBG] pgmap v224: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:28:54.739 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:28:54 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:28:53.232740+0000 mgr.smithi040.ogcprl (mgr.14164) 239 : cluster [DBG] pgmap v225: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:28:55.194 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:55.192+0000 7f8122e5c700 1 -- 172.21.15.66:0/3182641352 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f811c0f6480 msgr2=0x7f811c0f68a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:28:55.194 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:55.192+0000 7f8122e5c700 1 --2- 172.21.15.66:0/3182641352 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f811c0f6480 0x7f811c0f68a0 secure :-1 s=READY pgs=265 cs=0 l=1 rev1=1 rx=0x7f8110009fb0 tx=0x7f8110009b30).stop 2023-11-17T05:28:55.194 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:55.192+0000 7f8122e5c700 1 -- 172.21.15.66:0/3182641352 shutdown_connections 2023-11-17T05:28:55.194 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:55.192+0000 7f8122e5c700 1 --2- 172.21.15.66:0/3182641352 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f811c0f6480 0x7f811c0f68a0 unknown :-1 s=CLOSED pgs=265 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:28:55.194 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:55.192+0000 7f8122e5c700 1 -- 172.21.15.66:0/3182641352 >> 172.21.15.66:0/3182641352 conn(0x7f811c0f1c00 msgr2=0x7f811c0f4020 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:28:55.194 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:55.192+0000 7f8122e5c700 1 -- 172.21.15.66:0/3182641352 shutdown_connections 2023-11-17T05:28:55.194 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:55.192+0000 7f8122e5c700 1 -- 172.21.15.66:0/3182641352 wait complete. 2023-11-17T05:28:55.195 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:55.192+0000 7f8122e5c700 1 Processor -- start 2023-11-17T05:28:55.195 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:55.192+0000 7f8122e5c700 1 -- start start 2023-11-17T05:28:55.195 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:55.193+0000 7f8122e5c700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f811c0f6480 0x7f811c1066d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:28:55.195 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:55.193+0000 7f8122e5c700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f811c106bc0 con 0x7f811c0f6480 2023-11-17T05:28:55.195 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:55.193+0000 7f8120bf8700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f811c0f6480 0x7f811c1066d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:28:55.195 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:55.193+0000 7f8120bf8700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f811c0f6480 0x7f811c1066d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.66:60892/0 (socket says 172.21.15.66:60892) 2023-11-17T05:28:55.195 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:55.193+0000 7f8120bf8700 1 -- 172.21.15.66:0/2975476860 learned_addr learned my addr 172.21.15.66:0/2975476860 (peer_addr_for_me v2:172.21.15.66:0/0) 2023-11-17T05:28:55.196 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:55.193+0000 7f8120bf8700 1 -- 172.21.15.66:0/2975476860 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f8110005040 con 0x7f811c0f6480 2023-11-17T05:28:55.196 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:55.193+0000 7f8120bf8700 1 --2- 172.21.15.66:0/2975476860 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f811c0f6480 0x7f811c1066d0 secure :-1 s=READY pgs=266 cs=0 l=1 rev1=1 rx=0x7f8110013510 tx=0x7f811000b040).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:28:55.196 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:55.193+0000 7f81197fa700 1 -- 172.21.15.66:0/2975476860 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f8110009320 con 0x7f811c0f6480 2023-11-17T05:28:55.197 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:55.193+0000 7f8122e5c700 1 -- 172.21.15.66:0/2975476860 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f811c106dc0 con 0x7f811c0f6480 2023-11-17T05:28:55.197 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:55.193+0000 7f81197fa700 1 -- 172.21.15.66:0/2975476860 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(25 keys) v1 ==== 1074+0+0 (secure 0 0 0) 0x7f8110013f00 con 0x7f811c0f6480 2023-11-17T05:28:55.197 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:55.193+0000 7f81197fa700 1 -- 172.21.15.66:0/2975476860 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f8110014d90 con 0x7f811c0f6480 2023-11-17T05:28:55.197 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:55.194+0000 7f8122e5c700 1 -- 172.21.15.66:0/2975476860 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f811c107250 con 0x7f811c0f6480 2023-11-17T05:28:55.197 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:55.194+0000 7f81197fa700 1 -- 172.21.15.66:0/2975476860 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 17) v1 ==== 36269+0+0 (secure 0 0 0) 0x7f81100095f0 con 0x7f811c0f6480 2023-11-17T05:28:55.197 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:55.194+0000 7f8122e5c700 1 -- 172.21.15.66:0/2975476860 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f811c045ad0 con 0x7f811c0f6480 2023-11-17T05:28:55.197 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:55.194+0000 7f81197fa700 1 --2- 172.21.15.66:0/2975476860 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f810c02ff60 0x7f810c032420 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:28:55.197 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:55.195+0000 7f81197fa700 1 -- 172.21.15.66:0/2975476860 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(5..5 src has 1..5) v4 ==== 1270+0+0 (secure 0 0 0) 0x7f8110041000 con 0x7f811c0f6480 2023-11-17T05:28:55.197 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:55.195+0000 7f811bfff700 1 --2- 172.21.15.66:0/2975476860 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f810c02ff60 0x7f810c032420 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:28:55.197 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:55.195+0000 7f811bfff700 1 --2- 172.21.15.66:0/2975476860 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f810c02ff60 0x7f810c032420 secure :-1 s=READY pgs=98 cs=0 l=1 rev1=1 rx=0x7f81080096a0 tx=0x7f8108006b40).ready entity=mgr.14164 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:28:55.201 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:55.200+0000 7f81197fa700 1 -- 172.21.15.66:0/2975476860 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+152212 (secure 0 0 0) 0x7f8110014420 con 0x7f811c0f6480 2023-11-17T05:28:55.421 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:55.420+0000 7f8122e5c700 1 -- 172.21.15.66:0/2975476860 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "mon dump", "format": "json"} v 0) v1 -- 0x7f811c108100 con 0x7f811c0f6480 2023-11-17T05:28:55.422 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:55.420+0000 7f81197fa700 1 -- 172.21.15.66:0/2975476860 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mon_command_ack([{"prefix": "mon dump", "format": "json"}]=0 dumped monmap epoch 1 v1) v1 ==== 95+0+690 (secure 0 0 0) 0x7f811001d7f0 con 0x7f811c0f6480 2023-11-17T05:28:55.422 INFO:teuthology.orchestra.run.smithi066.stdout: 2023-11-17T05:28:55.422 INFO:teuthology.orchestra.run.smithi066.stdout:{"epoch":1,"fsid":"ed6470e2-8508-11ee-8dbe-212e2dc638e7","modified":"2023-11-17T05:20:13.934038Z","created":"2023-11-17T05:20:13.934038Z","min_mon_release":16,"min_mon_release_name":"pacific","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging"],"optional":[]},"mons":[{"rank":0,"name":"smithi040","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:3300","nonce":0},{"type":"v1","addr":"172.21.15.40:6789","nonce":0}]},"addr":"172.21.15.40:6789/0","public_addr":"172.21.15.40:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2023-11-17T05:28:55.423 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:55.422+0000 7f8122e5c700 1 -- 172.21.15.66:0/2975476860 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f810c02ff60 msgr2=0x7f810c032420 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:28:55.423 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:55.422+0000 7f8122e5c700 1 --2- 172.21.15.66:0/2975476860 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f810c02ff60 0x7f810c032420 secure :-1 s=READY pgs=98 cs=0 l=1 rev1=1 rx=0x7f81080096a0 tx=0x7f8108006b40).stop 2023-11-17T05:28:55.423 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:55.422+0000 7f8122e5c700 1 -- 172.21.15.66:0/2975476860 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f811c0f6480 msgr2=0x7f811c1066d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:28:55.423 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:55.422+0000 7f8122e5c700 1 --2- 172.21.15.66:0/2975476860 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f811c0f6480 0x7f811c1066d0 secure :-1 s=READY pgs=266 cs=0 l=1 rev1=1 rx=0x7f8110013510 tx=0x7f811000b040).stop 2023-11-17T05:28:55.423 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:55.422+0000 7f8122e5c700 1 -- 172.21.15.66:0/2975476860 shutdown_connections 2023-11-17T05:28:55.423 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:55.422+0000 7f8122e5c700 1 --2- 172.21.15.66:0/2975476860 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f810c02ff60 0x7f810c032420 unknown :-1 s=CLOSED pgs=98 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:28:55.423 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:55.422+0000 7f8122e5c700 1 --2- 172.21.15.66:0/2975476860 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f811c0f6480 0x7f811c1066d0 unknown :-1 s=CLOSED pgs=266 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:28:55.424 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:55.422+0000 7f8122e5c700 1 -- 172.21.15.66:0/2975476860 >> 172.21.15.66:0/2975476860 conn(0x7f811c0f1c00 msgr2=0x7f811c0f2720 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:28:55.424 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:55.422+0000 7f8122e5c700 1 -- 172.21.15.66:0/2975476860 shutdown_connections 2023-11-17T05:28:55.424 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:55.422+0000 7f8122e5c700 1 -- 172.21.15.66:0/2975476860 wait complete. 2023-11-17T05:28:55.424 INFO:teuthology.orchestra.run.smithi066.stderr:dumped monmap epoch 1 2023-11-17T05:28:55.988 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:28:55 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:28:55.422038+0000 mon.smithi040 (mon.0) 229 : audit [DBG] from='client.? 172.21.15.66:0/2975476860' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-11-17T05:28:56.841 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2023-11-17T05:28:56.841 DEBUG:teuthology.orchestra.run.smithi066:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.4 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ed6470e2-8508-11ee-8dbe-212e2dc638e7 -- ceph mon dump -f json 2023-11-17T05:28:56.988 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:28:56 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:28:55.232961+0000 mgr.smithi040.ogcprl (mgr.14164) 240 : cluster [DBG] pgmap v226: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:28:57.995 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:57.994+0000 7f9dca7d1700 1 -- 172.21.15.66:0/4010456539 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f9dc40f6510 msgr2=0x7f9dc40f6930 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:28:57.996 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:57.994+0000 7f9dca7d1700 1 --2- 172.21.15.66:0/4010456539 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f9dc40f6510 0x7f9dc40f6930 secure :-1 s=READY pgs=267 cs=0 l=1 rev1=1 rx=0x7f9db4009fb0 tx=0x7f9db4009b30).stop 2023-11-17T05:28:57.996 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:57.994+0000 7f9dca7d1700 1 -- 172.21.15.66:0/4010456539 shutdown_connections 2023-11-17T05:28:57.996 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:57.994+0000 7f9dca7d1700 1 --2- 172.21.15.66:0/4010456539 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f9dc40f6510 0x7f9dc40f6930 unknown :-1 s=CLOSED pgs=267 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:28:57.996 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:57.994+0000 7f9dca7d1700 1 -- 172.21.15.66:0/4010456539 >> 172.21.15.66:0/4010456539 conn(0x7f9dc40f1c70 msgr2=0x7f9dc40f4090 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:28:57.996 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:57.994+0000 7f9dca7d1700 1 -- 172.21.15.66:0/4010456539 shutdown_connections 2023-11-17T05:28:57.996 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:57.994+0000 7f9dca7d1700 1 -- 172.21.15.66:0/4010456539 wait complete. 2023-11-17T05:28:57.996 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:57.994+0000 7f9dca7d1700 1 Processor -- start 2023-11-17T05:28:57.998 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:57.995+0000 7f9dca7d1700 1 -- start start 2023-11-17T05:28:57.998 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:57.995+0000 7f9dca7d1700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f9dc40f6510 0x7f9dc4106730 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:28:57.998 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:57.995+0000 7f9dca7d1700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f9dc4106c20 con 0x7f9dc40f6510 2023-11-17T05:28:57.998 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:57.995+0000 7f9dc3fff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f9dc40f6510 0x7f9dc4106730 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:28:57.998 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:57.995+0000 7f9dc3fff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f9dc40f6510 0x7f9dc4106730 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.66:60910/0 (socket says 172.21.15.66:60910) 2023-11-17T05:28:57.998 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:57.995+0000 7f9dc3fff700 1 -- 172.21.15.66:0/596681675 learned_addr learned my addr 172.21.15.66:0/596681675 (peer_addr_for_me v2:172.21.15.66:0/0) 2023-11-17T05:28:57.998 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:57.996+0000 7f9dc3fff700 1 -- 172.21.15.66:0/596681675 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f9db4005040 con 0x7f9dc40f6510 2023-11-17T05:28:57.998 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:57.996+0000 7f9dc3fff700 1 --2- 172.21.15.66:0/596681675 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f9dc40f6510 0x7f9dc4106730 secure :-1 s=READY pgs=268 cs=0 l=1 rev1=1 rx=0x7f9db4013510 tx=0x7f9db400b040).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:28:57.998 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:57.996+0000 7f9dc0ff9700 1 -- 172.21.15.66:0/596681675 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f9db4009320 con 0x7f9dc40f6510 2023-11-17T05:28:57.999 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:57.996+0000 7f9dca7d1700 1 -- 172.21.15.66:0/596681675 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f9dc4106e20 con 0x7f9dc40f6510 2023-11-17T05:28:57.999 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:57.996+0000 7f9dc0ff9700 1 -- 172.21.15.66:0/596681675 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(25 keys) v1 ==== 1074+0+0 (secure 0 0 0) 0x7f9db4013f00 con 0x7f9dc40f6510 2023-11-17T05:28:57.999 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:57.996+0000 7f9dc0ff9700 1 -- 172.21.15.66:0/596681675 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f9db4014d90 con 0x7f9dc40f6510 2023-11-17T05:28:57.999 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:57.996+0000 7f9dca7d1700 1 -- 172.21.15.66:0/596681675 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f9dc41072b0 con 0x7f9dc40f6510 2023-11-17T05:28:58.000 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:57.997+0000 7f9dc0ff9700 1 -- 172.21.15.66:0/596681675 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 17) v1 ==== 36269+0+0 (secure 0 0 0) 0x7f9db40263a0 con 0x7f9dc40f6510 2023-11-17T05:28:58.000 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:57.997+0000 7f9dca7d1700 1 -- 172.21.15.66:0/596681675 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f9dc4045ad0 con 0x7f9dc40f6510 2023-11-17T05:28:58.000 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:57.997+0000 7f9dc0ff9700 1 --2- 172.21.15.66:0/596681675 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f9dac02ff60 0x7f9dac032420 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:28:58.000 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:57.997+0000 7f9dc0ff9700 1 -- 172.21.15.66:0/596681675 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(5..5 src has 1..5) v4 ==== 1270+0+0 (secure 0 0 0) 0x7f9db40410c0 con 0x7f9dc40f6510 2023-11-17T05:28:58.000 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:57.997+0000 7f9dc37fe700 1 --2- 172.21.15.66:0/596681675 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f9dac02ff60 0x7f9dac032420 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:28:58.000 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:57.998+0000 7f9dc37fe700 1 --2- 172.21.15.66:0/596681675 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f9dac02ff60 0x7f9dac032420 secure :-1 s=READY pgs=99 cs=0 l=1 rev1=1 rx=0x7f9db80096a0 tx=0x7f9db8006b40).ready entity=mgr.14164 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:28:58.003 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:58.002+0000 7f9dc0ff9700 1 -- 172.21.15.66:0/596681675 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+152212 (secure 0 0 0) 0x7f9db40095f0 con 0x7f9dc40f6510 2023-11-17T05:28:58.220 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:58.218+0000 7f9dca7d1700 1 -- 172.21.15.66:0/596681675 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "mon dump", "format": "json"} v 0) v1 -- 0x7f9dc40fbf60 con 0x7f9dc40f6510 2023-11-17T05:28:58.221 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:58.219+0000 7f9dc0ff9700 1 -- 172.21.15.66:0/596681675 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mon_command_ack([{"prefix": "mon dump", "format": "json"}]=0 dumped monmap epoch 1 v1) v1 ==== 95+0+690 (secure 0 0 0) 0x7f9db401db70 con 0x7f9dc40f6510 2023-11-17T05:28:58.221 INFO:teuthology.orchestra.run.smithi066.stdout: 2023-11-17T05:28:58.221 INFO:teuthology.orchestra.run.smithi066.stdout:{"epoch":1,"fsid":"ed6470e2-8508-11ee-8dbe-212e2dc638e7","modified":"2023-11-17T05:20:13.934038Z","created":"2023-11-17T05:20:13.934038Z","min_mon_release":16,"min_mon_release_name":"pacific","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging"],"optional":[]},"mons":[{"rank":0,"name":"smithi040","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:3300","nonce":0},{"type":"v1","addr":"172.21.15.40:6789","nonce":0}]},"addr":"172.21.15.40:6789/0","public_addr":"172.21.15.40:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2023-11-17T05:28:58.222 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:58.221+0000 7f9dca7d1700 1 -- 172.21.15.66:0/596681675 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f9dac02ff60 msgr2=0x7f9dac032420 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:28:58.222 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:58.221+0000 7f9dca7d1700 1 --2- 172.21.15.66:0/596681675 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f9dac02ff60 0x7f9dac032420 secure :-1 s=READY pgs=99 cs=0 l=1 rev1=1 rx=0x7f9db80096a0 tx=0x7f9db8006b40).stop 2023-11-17T05:28:58.223 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:58.221+0000 7f9dca7d1700 1 -- 172.21.15.66:0/596681675 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f9dc40f6510 msgr2=0x7f9dc4106730 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:28:58.223 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:58.221+0000 7f9dca7d1700 1 --2- 172.21.15.66:0/596681675 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f9dc40f6510 0x7f9dc4106730 secure :-1 s=READY pgs=268 cs=0 l=1 rev1=1 rx=0x7f9db4013510 tx=0x7f9db400b040).stop 2023-11-17T05:28:58.223 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:58.221+0000 7f9dca7d1700 1 -- 172.21.15.66:0/596681675 shutdown_connections 2023-11-17T05:28:58.223 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:58.221+0000 7f9dca7d1700 1 --2- 172.21.15.66:0/596681675 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f9dac02ff60 0x7f9dac032420 unknown :-1 s=CLOSED pgs=99 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:28:58.223 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:58.221+0000 7f9dca7d1700 1 --2- 172.21.15.66:0/596681675 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f9dc40f6510 0x7f9dc4106730 unknown :-1 s=CLOSED pgs=268 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:28:58.223 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:58.221+0000 7f9dca7d1700 1 -- 172.21.15.66:0/596681675 >> 172.21.15.66:0/596681675 conn(0x7f9dc40f1c70 msgr2=0x7f9dc40f2760 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:28:58.223 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:58.221+0000 7f9dca7d1700 1 -- 172.21.15.66:0/596681675 shutdown_connections 2023-11-17T05:28:58.223 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:28:58.221+0000 7f9dca7d1700 1 -- 172.21.15.66:0/596681675 wait complete. 2023-11-17T05:28:58.224 INFO:teuthology.orchestra.run.smithi066.stderr:dumped monmap epoch 1 2023-11-17T05:28:58.989 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:28:58 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:28:57.233207+0000 mgr.smithi040.ogcprl (mgr.14164) 241 : cluster [DBG] pgmap v227: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:28:58.989 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:28:58 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:28:58.220868+0000 mon.smithi040 (mon.0) 230 : audit [DBG] from='client.? 172.21.15.66:0/596681675' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-11-17T05:28:59.598 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2023-11-17T05:28:59.599 DEBUG:teuthology.orchestra.run.smithi066:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.4 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ed6470e2-8508-11ee-8dbe-212e2dc638e7 -- ceph mon dump -f json 2023-11-17T05:29:00.725 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:00.722+0000 7f2ef164b700 1 -- 172.21.15.66:0/1598433638 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f2eec0f6510 msgr2=0x7f2eec0f6930 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:29:00.725 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:00.722+0000 7f2ef164b700 1 --2- 172.21.15.66:0/1598433638 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f2eec0f6510 0x7f2eec0f6930 secure :-1 s=READY pgs=269 cs=0 l=1 rev1=1 rx=0x7f2edc009fb0 tx=0x7f2edc009b30).stop 2023-11-17T05:29:00.725 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:00.722+0000 7f2ef164b700 1 -- 172.21.15.66:0/1598433638 shutdown_connections 2023-11-17T05:29:00.725 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:00.722+0000 7f2ef164b700 1 --2- 172.21.15.66:0/1598433638 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f2eec0f6510 0x7f2eec0f6930 unknown :-1 s=CLOSED pgs=269 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:29:00.725 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:00.722+0000 7f2ef164b700 1 -- 172.21.15.66:0/1598433638 >> 172.21.15.66:0/1598433638 conn(0x7f2eec0f1c70 msgr2=0x7f2eec0f4090 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:29:00.725 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:00.722+0000 7f2ef164b700 1 -- 172.21.15.66:0/1598433638 shutdown_connections 2023-11-17T05:29:00.725 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:00.722+0000 7f2ef164b700 1 -- 172.21.15.66:0/1598433638 wait complete. 2023-11-17T05:29:00.725 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:00.723+0000 7f2ef164b700 1 Processor -- start 2023-11-17T05:29:00.725 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:00.723+0000 7f2ef164b700 1 -- start start 2023-11-17T05:29:00.726 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:00.724+0000 7f2ef164b700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f2eec0f6510 0x7f2eec1088d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:29:00.726 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:00.724+0000 7f2ef164b700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f2eec108dc0 con 0x7f2eec0f6510 2023-11-17T05:29:00.728 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:00.724+0000 7f2eeaffd700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f2eec0f6510 0x7f2eec1088d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:29:00.728 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:00.724+0000 7f2eeaffd700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f2eec0f6510 0x7f2eec1088d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.66:60922/0 (socket says 172.21.15.66:60922) 2023-11-17T05:29:00.728 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:00.724+0000 7f2eeaffd700 1 -- 172.21.15.66:0/3301022671 learned_addr learned my addr 172.21.15.66:0/3301022671 (peer_addr_for_me v2:172.21.15.66:0/0) 2023-11-17T05:29:00.728 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:00.724+0000 7f2eeaffd700 1 -- 172.21.15.66:0/3301022671 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f2edc005040 con 0x7f2eec0f6510 2023-11-17T05:29:00.728 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:00.724+0000 7f2eeaffd700 1 --2- 172.21.15.66:0/3301022671 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f2eec0f6510 0x7f2eec1088d0 secure :-1 s=READY pgs=270 cs=0 l=1 rev1=1 rx=0x7f2edc013510 tx=0x7f2edc00b040).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:29:00.728 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:00.724+0000 7f2ee37fe700 1 -- 172.21.15.66:0/3301022671 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f2edc009320 con 0x7f2eec0f6510 2023-11-17T05:29:00.728 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:00.724+0000 7f2ee37fe700 1 -- 172.21.15.66:0/3301022671 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(25 keys) v1 ==== 1074+0+0 (secure 0 0 0) 0x7f2edc013f00 con 0x7f2eec0f6510 2023-11-17T05:29:00.729 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:00.724+0000 7f2ef164b700 1 -- 172.21.15.66:0/3301022671 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f2eec108fc0 con 0x7f2eec0f6510 2023-11-17T05:29:00.729 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:00.724+0000 7f2ee37fe700 1 -- 172.21.15.66:0/3301022671 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f2edc014d90 con 0x7f2eec0f6510 2023-11-17T05:29:00.729 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:00.725+0000 7f2ef164b700 1 -- 172.21.15.66:0/3301022671 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f2eec109450 con 0x7f2eec0f6510 2023-11-17T05:29:00.729 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:00.725+0000 7f2ee37fe700 1 -- 172.21.15.66:0/3301022671 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 17) v1 ==== 36269+0+0 (secure 0 0 0) 0x7f2edc0095f0 con 0x7f2eec0f6510 2023-11-17T05:29:00.729 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:00.725+0000 7f2ef164b700 1 -- 172.21.15.66:0/3301022671 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f2eec045ad0 con 0x7f2eec0f6510 2023-11-17T05:29:00.729 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:00.725+0000 7f2ee37fe700 1 --2- 172.21.15.66:0/3301022671 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f2ed802ff60 0x7f2ed8032420 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:29:00.729 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:00.725+0000 7f2ee37fe700 1 -- 172.21.15.66:0/3301022671 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(5..5 src has 1..5) v4 ==== 1270+0+0 (secure 0 0 0) 0x7f2edc040fe0 con 0x7f2eec0f6510 2023-11-17T05:29:00.729 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:00.726+0000 7f2eea7fc700 1 --2- 172.21.15.66:0/3301022671 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f2ed802ff60 0x7f2ed8032420 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:29:00.729 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:00.726+0000 7f2eea7fc700 1 --2- 172.21.15.66:0/3301022671 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f2ed802ff60 0x7f2ed8032420 secure :-1 s=READY pgs=100 cs=0 l=1 rev1=1 rx=0x7f2ed40096a0 tx=0x7f2ed4006b40).ready entity=mgr.14164 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:29:00.732 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:00.730+0000 7f2ee37fe700 1 -- 172.21.15.66:0/3301022671 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+152212 (secure 0 0 0) 0x7f2edc01d7f0 con 0x7f2eec0f6510 2023-11-17T05:29:00.952 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:00.951+0000 7f2ef164b700 1 -- 172.21.15.66:0/3301022671 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "mon dump", "format": "json"} v 0) v1 -- 0x7f2eec06b280 con 0x7f2eec0f6510 2023-11-17T05:29:00.953 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:00.951+0000 7f2ee37fe700 1 -- 172.21.15.66:0/3301022671 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mon_command_ack([{"prefix": "mon dump", "format": "json"}]=0 dumped monmap epoch 1 v1) v1 ==== 95+0+690 (secure 0 0 0) 0x7f2edc010230 con 0x7f2eec0f6510 2023-11-17T05:29:00.953 INFO:teuthology.orchestra.run.smithi066.stdout: 2023-11-17T05:29:00.953 INFO:teuthology.orchestra.run.smithi066.stdout:{"epoch":1,"fsid":"ed6470e2-8508-11ee-8dbe-212e2dc638e7","modified":"2023-11-17T05:20:13.934038Z","created":"2023-11-17T05:20:13.934038Z","min_mon_release":16,"min_mon_release_name":"pacific","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging"],"optional":[]},"mons":[{"rank":0,"name":"smithi040","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:3300","nonce":0},{"type":"v1","addr":"172.21.15.40:6789","nonce":0}]},"addr":"172.21.15.40:6789/0","public_addr":"172.21.15.40:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2023-11-17T05:29:00.957 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:00.953+0000 7f2ef164b700 1 -- 172.21.15.66:0/3301022671 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f2ed802ff60 msgr2=0x7f2ed8032420 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:29:00.957 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:00.953+0000 7f2ef164b700 1 --2- 172.21.15.66:0/3301022671 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f2ed802ff60 0x7f2ed8032420 secure :-1 s=READY pgs=100 cs=0 l=1 rev1=1 rx=0x7f2ed40096a0 tx=0x7f2ed4006b40).stop 2023-11-17T05:29:00.957 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:00.953+0000 7f2ef164b700 1 -- 172.21.15.66:0/3301022671 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f2eec0f6510 msgr2=0x7f2eec1088d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:29:00.957 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:00.953+0000 7f2ef164b700 1 --2- 172.21.15.66:0/3301022671 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f2eec0f6510 0x7f2eec1088d0 secure :-1 s=READY pgs=270 cs=0 l=1 rev1=1 rx=0x7f2edc013510 tx=0x7f2edc00b040).stop 2023-11-17T05:29:00.957 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:00.953+0000 7f2ef164b700 1 -- 172.21.15.66:0/3301022671 shutdown_connections 2023-11-17T05:29:00.958 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:00.953+0000 7f2ef164b700 1 --2- 172.21.15.66:0/3301022671 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f2ed802ff60 0x7f2ed8032420 unknown :-1 s=CLOSED pgs=100 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:29:00.958 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:00.953+0000 7f2ef164b700 1 --2- 172.21.15.66:0/3301022671 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f2eec0f6510 0x7f2eec1088d0 unknown :-1 s=CLOSED pgs=270 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:29:00.958 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:00.953+0000 7f2ef164b700 1 -- 172.21.15.66:0/3301022671 >> 172.21.15.66:0/3301022671 conn(0x7f2eec0f1c70 msgr2=0x7f2eec0f4000 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:29:00.958 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:00.953+0000 7f2ef164b700 1 -- 172.21.15.66:0/3301022671 shutdown_connections 2023-11-17T05:29:00.958 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:00.953+0000 7f2ef164b700 1 -- 172.21.15.66:0/3301022671 wait complete. 2023-11-17T05:29:00.958 INFO:teuthology.orchestra.run.smithi066.stderr:dumped monmap epoch 1 2023-11-17T05:29:00.989 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:29:00 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:28:59.233528+0000 mgr.smithi040.ogcprl (mgr.14164) 242 : cluster [DBG] pgmap v228: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:29:01.989 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:29:01 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:29:00.953090+0000 mon.smithi040 (mon.0) 231 : audit [DBG] from='client.? 172.21.15.66:0/3301022671' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-11-17T05:29:02.347 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2023-11-17T05:29:02.347 DEBUG:teuthology.orchestra.run.smithi066:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.4 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ed6470e2-8508-11ee-8dbe-212e2dc638e7 -- ceph mon dump -f json 2023-11-17T05:29:02.988 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:29:02 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:29:01.233931+0000 mgr.smithi040.ogcprl (mgr.14164) 243 : cluster [DBG] pgmap v229: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:29:03.582 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:03.580+0000 7fa81d666700 1 -- 172.21.15.66:0/3010569016 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fa8180f47c0 msgr2=0x7fa8180f4be0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:29:03.583 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:03.580+0000 7fa81d666700 1 --2- 172.21.15.66:0/3010569016 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fa8180f47c0 0x7fa8180f4be0 secure :-1 s=READY pgs=271 cs=0 l=1 rev1=1 rx=0x7fa80c009fb0 tx=0x7fa80c009b30).stop 2023-11-17T05:29:03.583 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:03.580+0000 7fa81d666700 1 -- 172.21.15.66:0/3010569016 shutdown_connections 2023-11-17T05:29:03.583 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:03.580+0000 7fa81d666700 1 --2- 172.21.15.66:0/3010569016 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fa8180f47c0 0x7fa8180f4be0 unknown :-1 s=CLOSED pgs=271 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:29:03.583 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:03.580+0000 7fa81d666700 1 -- 172.21.15.66:0/3010569016 >> 172.21.15.66:0/3010569016 conn(0x7fa8180f02e0 msgr2=0x7fa8180f2700 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:29:03.583 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:03.580+0000 7fa81d666700 1 -- 172.21.15.66:0/3010569016 shutdown_connections 2023-11-17T05:29:03.583 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:03.580+0000 7fa81d666700 1 -- 172.21.15.66:0/3010569016 wait complete. 2023-11-17T05:29:03.583 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:03.581+0000 7fa81d666700 1 Processor -- start 2023-11-17T05:29:03.583 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:03.581+0000 7fa81d666700 1 -- start start 2023-11-17T05:29:03.583 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:03.581+0000 7fa81d666700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fa8180f47c0 0x7fa8180f7370 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:29:03.583 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:03.581+0000 7fa81d666700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fa8180f57f0 con 0x7fa8180f47c0 2023-11-17T05:29:03.585 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:03.582+0000 7fa816ffd700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fa8180f47c0 0x7fa8180f7370 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:29:03.585 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:03.582+0000 7fa816ffd700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fa8180f47c0 0x7fa8180f7370 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.66:52336/0 (socket says 172.21.15.66:52336) 2023-11-17T05:29:03.585 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:03.582+0000 7fa816ffd700 1 -- 172.21.15.66:0/2176271961 learned_addr learned my addr 172.21.15.66:0/2176271961 (peer_addr_for_me v2:172.21.15.66:0/0) 2023-11-17T05:29:03.585 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:03.582+0000 7fa816ffd700 1 -- 172.21.15.66:0/2176271961 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fa80c005040 con 0x7fa8180f47c0 2023-11-17T05:29:03.585 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:03.582+0000 7fa816ffd700 1 --2- 172.21.15.66:0/2176271961 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fa8180f47c0 0x7fa8180f7370 secure :-1 s=READY pgs=272 cs=0 l=1 rev1=1 rx=0x7fa80c014000 tx=0x7fa80c00b040).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:29:03.585 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:03.582+0000 7fa7ff7fe700 1 -- 172.21.15.66:0/2176271961 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7fa80c0095f0 con 0x7fa8180f47c0 2023-11-17T05:29:03.585 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:03.582+0000 7fa81d666700 1 -- 172.21.15.66:0/2176271961 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fa8180f5990 con 0x7fa8180f47c0 2023-11-17T05:29:03.585 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:03.582+0000 7fa7ff7fe700 1 -- 172.21.15.66:0/2176271961 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(25 keys) v1 ==== 1074+0+0 (secure 0 0 0) 0x7fa80c007e70 con 0x7fa8180f47c0 2023-11-17T05:29:03.585 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:03.582+0000 7fa7ff7fe700 1 -- 172.21.15.66:0/2176271961 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7fa80c014ea0 con 0x7fa8180f47c0 2023-11-17T05:29:03.585 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:03.582+0000 7fa81d666700 1 -- 172.21.15.66:0/2176271961 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fa8180f5e20 con 0x7fa8180f47c0 2023-11-17T05:29:03.585 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:03.583+0000 7fa81d666700 1 -- 172.21.15.66:0/2176271961 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fa818059100 con 0x7fa8180f47c0 2023-11-17T05:29:03.586 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:03.583+0000 7fa7ff7fe700 1 -- 172.21.15.66:0/2176271961 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 17) v1 ==== 36269+0+0 (secure 0 0 0) 0x7fa80c025070 con 0x7fa8180f47c0 2023-11-17T05:29:03.586 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:03.583+0000 7fa7ff7fe700 1 --2- 172.21.15.66:0/2176271961 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fa80002ff60 0x7fa800032420 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:29:03.586 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:03.583+0000 7fa7ff7fe700 1 -- 172.21.15.66:0/2176271961 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(5..5 src has 1..5) v4 ==== 1270+0+0 (secure 0 0 0) 0x7fa80c040950 con 0x7fa8180f47c0 2023-11-17T05:29:03.586 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:03.584+0000 7fa8167fc700 1 --2- 172.21.15.66:0/2176271961 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fa80002ff60 0x7fa800032420 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:29:03.586 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:03.584+0000 7fa8167fc700 1 --2- 172.21.15.66:0/2176271961 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fa80002ff60 0x7fa800032420 secure :-1 s=READY pgs=101 cs=0 l=1 rev1=1 rx=0x7fa8080096a0 tx=0x7fa808006b40).ready entity=mgr.14164 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:29:03.589 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:03.588+0000 7fa7ff7fe700 1 -- 172.21.15.66:0/2176271961 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+152212 (secure 0 0 0) 0x7fa80c0079c0 con 0x7fa8180f47c0 2023-11-17T05:29:03.807 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:03.806+0000 7fa81d666700 1 -- 172.21.15.66:0/2176271961 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "mon dump", "format": "json"} v 0) v1 -- 0x7fa818045ad0 con 0x7fa8180f47c0 2023-11-17T05:29:03.808 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:03.807+0000 7fa7ff7fe700 1 -- 172.21.15.66:0/2176271961 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mon_command_ack([{"prefix": "mon dump", "format": "json"}]=0 dumped monmap epoch 1 v1) v1 ==== 95+0+690 (secure 0 0 0) 0x7fa80c00f300 con 0x7fa8180f47c0 2023-11-17T05:29:03.808 INFO:teuthology.orchestra.run.smithi066.stdout: 2023-11-17T05:29:03.808 INFO:teuthology.orchestra.run.smithi066.stdout:{"epoch":1,"fsid":"ed6470e2-8508-11ee-8dbe-212e2dc638e7","modified":"2023-11-17T05:20:13.934038Z","created":"2023-11-17T05:20:13.934038Z","min_mon_release":16,"min_mon_release_name":"pacific","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging"],"optional":[]},"mons":[{"rank":0,"name":"smithi040","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:3300","nonce":0},{"type":"v1","addr":"172.21.15.40:6789","nonce":0}]},"addr":"172.21.15.40:6789/0","public_addr":"172.21.15.40:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2023-11-17T05:29:03.810 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:03.808+0000 7fa81d666700 1 -- 172.21.15.66:0/2176271961 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fa80002ff60 msgr2=0x7fa800032420 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:29:03.810 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:03.808+0000 7fa81d666700 1 --2- 172.21.15.66:0/2176271961 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fa80002ff60 0x7fa800032420 secure :-1 s=READY pgs=101 cs=0 l=1 rev1=1 rx=0x7fa8080096a0 tx=0x7fa808006b40).stop 2023-11-17T05:29:03.810 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:03.808+0000 7fa81d666700 1 -- 172.21.15.66:0/2176271961 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fa8180f47c0 msgr2=0x7fa8180f7370 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:29:03.810 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:03.808+0000 7fa81d666700 1 --2- 172.21.15.66:0/2176271961 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fa8180f47c0 0x7fa8180f7370 secure :-1 s=READY pgs=272 cs=0 l=1 rev1=1 rx=0x7fa80c014000 tx=0x7fa80c00b040).stop 2023-11-17T05:29:03.810 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:03.808+0000 7fa81d666700 1 -- 172.21.15.66:0/2176271961 shutdown_connections 2023-11-17T05:29:03.810 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:03.808+0000 7fa81d666700 1 --2- 172.21.15.66:0/2176271961 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fa80002ff60 0x7fa800032420 unknown :-1 s=CLOSED pgs=101 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:29:03.810 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:03.808+0000 7fa81d666700 1 --2- 172.21.15.66:0/2176271961 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fa8180f47c0 0x7fa8180f7370 unknown :-1 s=CLOSED pgs=272 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:29:03.810 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:03.808+0000 7fa81d666700 1 -- 172.21.15.66:0/2176271961 >> 172.21.15.66:0/2176271961 conn(0x7fa8180f02e0 msgr2=0x7fa8180f0db0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:29:03.810 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:03.808+0000 7fa81d666700 1 -- 172.21.15.66:0/2176271961 shutdown_connections 2023-11-17T05:29:03.810 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:03.808+0000 7fa81d666700 1 -- 172.21.15.66:0/2176271961 wait complete. 2023-11-17T05:29:03.811 INFO:teuthology.orchestra.run.smithi066.stderr:dumped monmap epoch 1 2023-11-17T05:29:04.989 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:29:04 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:29:03.234249+0000 mgr.smithi040.ogcprl (mgr.14164) 244 : cluster [DBG] pgmap v230: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:29:04.989 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:29:04 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:29:03.808359+0000 mon.smithi040 (mon.0) 232 : audit [DBG] from='client.? 172.21.15.66:0/2176271961' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-11-17T05:29:05.234 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2023-11-17T05:29:05.235 DEBUG:teuthology.orchestra.run.smithi066:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.4 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ed6470e2-8508-11ee-8dbe-212e2dc638e7 -- ceph mon dump -f json 2023-11-17T05:29:06.386 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:06.384+0000 7f627c350700 1 -- 172.21.15.66:0/3402398535 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f62740f6510 msgr2=0x7f62740f6930 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:29:06.387 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:06.384+0000 7f627c350700 1 --2- 172.21.15.66:0/3402398535 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f62740f6510 0x7f62740f6930 secure :-1 s=READY pgs=273 cs=0 l=1 rev1=1 rx=0x7f6264009960 tx=0x7f6264004d10).stop 2023-11-17T05:29:06.387 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:06.384+0000 7f627c350700 1 -- 172.21.15.66:0/3402398535 shutdown_connections 2023-11-17T05:29:06.387 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:06.384+0000 7f627c350700 1 --2- 172.21.15.66:0/3402398535 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f62740f6510 0x7f62740f6930 unknown :-1 s=CLOSED pgs=273 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:29:06.387 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:06.384+0000 7f627c350700 1 -- 172.21.15.66:0/3402398535 >> 172.21.15.66:0/3402398535 conn(0x7f62740f1c70 msgr2=0x7f62740f4090 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:29:06.387 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:06.384+0000 7f627c350700 1 -- 172.21.15.66:0/3402398535 shutdown_connections 2023-11-17T05:29:06.387 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:06.384+0000 7f627c350700 1 -- 172.21.15.66:0/3402398535 wait complete. 2023-11-17T05:29:06.387 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:06.385+0000 7f627c350700 1 Processor -- start 2023-11-17T05:29:06.387 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:06.385+0000 7f627c350700 1 -- start start 2023-11-17T05:29:06.387 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:06.385+0000 7f627c350700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f62740f6510 0x7f627406b9a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:29:06.387 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:06.385+0000 7f627c350700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f6274069e20 con 0x7f62740f6510 2023-11-17T05:29:06.387 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:06.385+0000 7f627a0ec700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f62740f6510 0x7f627406b9a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:29:06.388 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:06.385+0000 7f627a0ec700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f62740f6510 0x7f627406b9a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.66:52356/0 (socket says 172.21.15.66:52356) 2023-11-17T05:29:06.388 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:06.385+0000 7f627a0ec700 1 -- 172.21.15.66:0/4012408159 learned_addr learned my addr 172.21.15.66:0/4012408159 (peer_addr_for_me v2:172.21.15.66:0/0) 2023-11-17T05:29:06.389 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:06.386+0000 7f627a0ec700 1 -- 172.21.15.66:0/4012408159 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f6264009640 con 0x7f62740f6510 2023-11-17T05:29:06.389 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:06.386+0000 7f627a0ec700 1 --2- 172.21.15.66:0/4012408159 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f62740f6510 0x7f627406b9a0 secure :-1 s=READY pgs=274 cs=0 l=1 rev1=1 rx=0x7f6264004b70 tx=0x7f6264004d10).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:29:06.389 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:06.386+0000 7f6272ffd700 1 -- 172.21.15.66:0/4012408159 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f6264018070 con 0x7f62740f6510 2023-11-17T05:29:06.389 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:06.386+0000 7f627c350700 1 -- 172.21.15.66:0/4012408159 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f6274069fc0 con 0x7f62740f6510 2023-11-17T05:29:06.389 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:06.386+0000 7f6272ffd700 1 -- 172.21.15.66:0/4012408159 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(25 keys) v1 ==== 1074+0+0 (secure 0 0 0) 0x7f626400b870 con 0x7f62740f6510 2023-11-17T05:29:06.389 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:06.386+0000 7f6272ffd700 1 -- 172.21.15.66:0/4012408159 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f626401c560 con 0x7f62740f6510 2023-11-17T05:29:06.390 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:06.386+0000 7f627c350700 1 -- 172.21.15.66:0/4012408159 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f627406a450 con 0x7f62740f6510 2023-11-17T05:29:06.390 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:06.387+0000 7f6272ffd700 1 -- 172.21.15.66:0/4012408159 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 17) v1 ==== 36269+0+0 (secure 0 0 0) 0x7f6264013630 con 0x7f62740f6510 2023-11-17T05:29:06.390 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:06.387+0000 7f627c350700 1 -- 172.21.15.66:0/4012408159 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f6274059100 con 0x7f62740f6510 2023-11-17T05:29:06.390 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:06.387+0000 7f6272ffd700 1 --2- 172.21.15.66:0/4012408159 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f625c02ffb0 0x7f625c032470 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:29:06.390 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:06.387+0000 7f6272ffd700 1 -- 172.21.15.66:0/4012408159 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(5..5 src has 1..5) v4 ==== 1270+0+0 (secure 0 0 0) 0x7f62640411b0 con 0x7f62740f6510 2023-11-17T05:29:06.390 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:06.387+0000 7f62798eb700 1 --2- 172.21.15.66:0/4012408159 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f625c02ffb0 0x7f625c032470 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:29:06.390 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:06.388+0000 7f62798eb700 1 --2- 172.21.15.66:0/4012408159 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f625c02ffb0 0x7f625c032470 secure :-1 s=READY pgs=102 cs=0 l=1 rev1=1 rx=0x7f62680096a0 tx=0x7f6268006b40).ready entity=mgr.14164 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:29:06.393 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:06.392+0000 7f6272ffd700 1 -- 172.21.15.66:0/4012408159 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+152212 (secure 0 0 0) 0x7f6264013a30 con 0x7f62740f6510 2023-11-17T05:29:06.614 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:06.613+0000 7f627c350700 1 -- 172.21.15.66:0/4012408159 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "mon dump", "format": "json"} v 0) v1 -- 0x7f62740fbf60 con 0x7f62740f6510 2023-11-17T05:29:06.615 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:06.614+0000 7f6272ffd700 1 -- 172.21.15.66:0/4012408159 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mon_command_ack([{"prefix": "mon dump", "format": "json"}]=0 dumped monmap epoch 1 v1) v1 ==== 95+0+690 (secure 0 0 0) 0x7f6264011b10 con 0x7f62740f6510 2023-11-17T05:29:06.615 INFO:teuthology.orchestra.run.smithi066.stdout: 2023-11-17T05:29:06.615 INFO:teuthology.orchestra.run.smithi066.stdout:{"epoch":1,"fsid":"ed6470e2-8508-11ee-8dbe-212e2dc638e7","modified":"2023-11-17T05:20:13.934038Z","created":"2023-11-17T05:20:13.934038Z","min_mon_release":16,"min_mon_release_name":"pacific","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging"],"optional":[]},"mons":[{"rank":0,"name":"smithi040","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:3300","nonce":0},{"type":"v1","addr":"172.21.15.40:6789","nonce":0}]},"addr":"172.21.15.40:6789/0","public_addr":"172.21.15.40:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2023-11-17T05:29:06.617 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:06.615+0000 7f627c350700 1 -- 172.21.15.66:0/4012408159 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f625c02ffb0 msgr2=0x7f625c032470 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:29:06.617 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:06.615+0000 7f627c350700 1 --2- 172.21.15.66:0/4012408159 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f625c02ffb0 0x7f625c032470 secure :-1 s=READY pgs=102 cs=0 l=1 rev1=1 rx=0x7f62680096a0 tx=0x7f6268006b40).stop 2023-11-17T05:29:06.617 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:06.615+0000 7f627c350700 1 -- 172.21.15.66:0/4012408159 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f62740f6510 msgr2=0x7f627406b9a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:29:06.617 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:06.615+0000 7f627c350700 1 --2- 172.21.15.66:0/4012408159 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f62740f6510 0x7f627406b9a0 secure :-1 s=READY pgs=274 cs=0 l=1 rev1=1 rx=0x7f6264004b70 tx=0x7f6264004d10).stop 2023-11-17T05:29:06.617 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:06.615+0000 7f627c350700 1 -- 172.21.15.66:0/4012408159 shutdown_connections 2023-11-17T05:29:06.617 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:06.615+0000 7f627c350700 1 --2- 172.21.15.66:0/4012408159 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f625c02ffb0 0x7f625c032470 unknown :-1 s=CLOSED pgs=102 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:29:06.617 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:06.615+0000 7f627c350700 1 --2- 172.21.15.66:0/4012408159 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f62740f6510 0x7f627406b9a0 unknown :-1 s=CLOSED pgs=274 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:29:06.617 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:06.615+0000 7f627c350700 1 -- 172.21.15.66:0/4012408159 >> 172.21.15.66:0/4012408159 conn(0x7f62740f1c70 msgr2=0x7f62740f2760 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:29:06.617 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:06.616+0000 7f627c350700 1 -- 172.21.15.66:0/4012408159 shutdown_connections 2023-11-17T05:29:06.617 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:06.616+0000 7f627c350700 1 -- 172.21.15.66:0/4012408159 wait complete. 2023-11-17T05:29:06.618 INFO:teuthology.orchestra.run.smithi066.stderr:dumped monmap epoch 1 2023-11-17T05:29:06.988 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:29:06 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:29:05.234508+0000 mgr.smithi040.ogcprl (mgr.14164) 245 : cluster [DBG] pgmap v231: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:29:07.989 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:29:07 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:29:06.615325+0000 mon.smithi040 (mon.0) 233 : audit [DBG] from='client.? 172.21.15.66:0/4012408159' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-11-17T05:29:08.002 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2023-11-17T05:29:08.003 DEBUG:teuthology.orchestra.run.smithi066:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.4 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ed6470e2-8508-11ee-8dbe-212e2dc638e7 -- ceph mon dump -f json 2023-11-17T05:29:08.943 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:29:08 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:29:07.234873+0000 mgr.smithi040.ogcprl (mgr.14164) 246 : cluster [DBG] pgmap v232: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:29:09.154 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:09.151+0000 7f0edab0c700 1 -- 172.21.15.66:0/1583529075 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f0ed40f6510 msgr2=0x7f0ed40f6930 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:29:09.154 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:09.151+0000 7f0edab0c700 1 --2- 172.21.15.66:0/1583529075 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f0ed40f6510 0x7f0ed40f6930 secure :-1 s=READY pgs=275 cs=0 l=1 rev1=1 rx=0x7f0ec8009fb0 tx=0x7f0ec8009b30).stop 2023-11-17T05:29:09.154 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:09.151+0000 7f0edab0c700 1 -- 172.21.15.66:0/1583529075 shutdown_connections 2023-11-17T05:29:09.154 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:09.151+0000 7f0edab0c700 1 --2- 172.21.15.66:0/1583529075 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f0ed40f6510 0x7f0ed40f6930 unknown :-1 s=CLOSED pgs=275 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:29:09.154 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:09.151+0000 7f0edab0c700 1 -- 172.21.15.66:0/1583529075 >> 172.21.15.66:0/1583529075 conn(0x7f0ed40f1c70 msgr2=0x7f0ed40f4090 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:29:09.154 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:09.151+0000 7f0edab0c700 1 -- 172.21.15.66:0/1583529075 shutdown_connections 2023-11-17T05:29:09.154 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:09.151+0000 7f0edab0c700 1 -- 172.21.15.66:0/1583529075 wait complete. 2023-11-17T05:29:09.154 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:09.152+0000 7f0edab0c700 1 Processor -- start 2023-11-17T05:29:09.155 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:09.152+0000 7f0edab0c700 1 -- start start 2023-11-17T05:29:09.155 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:09.152+0000 7f0edab0c700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f0ed40f6510 0x7f0ed406b9a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:29:09.155 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:09.152+0000 7f0edab0c700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f0ed4069e20 con 0x7f0ed40f6510 2023-11-17T05:29:09.155 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:09.152+0000 7f0ed88a8700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f0ed40f6510 0x7f0ed406b9a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:29:09.155 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:09.153+0000 7f0ed88a8700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f0ed40f6510 0x7f0ed406b9a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.66:52376/0 (socket says 172.21.15.66:52376) 2023-11-17T05:29:09.155 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:09.153+0000 7f0ed88a8700 1 -- 172.21.15.66:0/409860994 learned_addr learned my addr 172.21.15.66:0/409860994 (peer_addr_for_me v2:172.21.15.66:0/0) 2023-11-17T05:29:09.156 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:09.153+0000 7f0ed88a8700 1 -- 172.21.15.66:0/409860994 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f0ec8005040 con 0x7f0ed40f6510 2023-11-17T05:29:09.157 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:09.153+0000 7f0ed88a8700 1 --2- 172.21.15.66:0/409860994 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f0ed40f6510 0x7f0ed406b9a0 secure :-1 s=READY pgs=276 cs=0 l=1 rev1=1 rx=0x7f0ec8013510 tx=0x7f0ec800b040).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:29:09.157 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:09.153+0000 7f0ed17fa700 1 -- 172.21.15.66:0/409860994 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f0ec8009320 con 0x7f0ed40f6510 2023-11-17T05:29:09.157 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:09.153+0000 7f0edab0c700 1 -- 172.21.15.66:0/409860994 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f0ed4069fc0 con 0x7f0ed40f6510 2023-11-17T05:29:09.157 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:09.153+0000 7f0ed17fa700 1 -- 172.21.15.66:0/409860994 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(25 keys) v1 ==== 1074+0+0 (secure 0 0 0) 0x7f0ec8013f00 con 0x7f0ed40f6510 2023-11-17T05:29:09.157 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:09.153+0000 7f0ed17fa700 1 -- 172.21.15.66:0/409860994 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f0ec8014d90 con 0x7f0ed40f6510 2023-11-17T05:29:09.157 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:09.153+0000 7f0edab0c700 1 -- 172.21.15.66:0/409860994 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f0ed406a450 con 0x7f0ed40f6510 2023-11-17T05:29:09.157 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:09.154+0000 7f0ed17fa700 1 -- 172.21.15.66:0/409860994 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 17) v1 ==== 36269+0+0 (secure 0 0 0) 0x7f0ec80095f0 con 0x7f0ed40f6510 2023-11-17T05:29:09.157 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:09.154+0000 7f0edab0c700 1 -- 172.21.15.66:0/409860994 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f0ed4059100 con 0x7f0ed40f6510 2023-11-17T05:29:09.157 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:09.154+0000 7f0ed17fa700 1 --2- 172.21.15.66:0/409860994 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f0ec402ffb0 0x7f0ec4032470 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:29:09.158 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:09.154+0000 7f0ed17fa700 1 -- 172.21.15.66:0/409860994 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(5..5 src has 1..5) v4 ==== 1270+0+0 (secure 0 0 0) 0x7f0ec80410c0 con 0x7f0ed40f6510 2023-11-17T05:29:09.158 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:09.155+0000 7f0ed3fff700 1 --2- 172.21.15.66:0/409860994 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f0ec402ffb0 0x7f0ec4032470 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:29:09.158 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:09.155+0000 7f0ed3fff700 1 --2- 172.21.15.66:0/409860994 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f0ec402ffb0 0x7f0ec4032470 secure :-1 s=READY pgs=103 cs=0 l=1 rev1=1 rx=0x7f0ec00096a0 tx=0x7f0ec0006b40).ready entity=mgr.14164 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:29:09.160 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:09.159+0000 7f0ed17fa700 1 -- 172.21.15.66:0/409860994 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+152212 (secure 0 0 0) 0x7f0ec8014420 con 0x7f0ed40f6510 2023-11-17T05:29:09.382 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:09.380+0000 7f0edab0c700 1 -- 172.21.15.66:0/409860994 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "mon dump", "format": "json"} v 0) v1 -- 0x7f0ed40fbf60 con 0x7f0ed40f6510 2023-11-17T05:29:09.382 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:09.381+0000 7f0ed17fa700 1 -- 172.21.15.66:0/409860994 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mon_command_ack([{"prefix": "mon dump", "format": "json"}]=0 dumped monmap epoch 1 v1) v1 ==== 95+0+690 (secure 0 0 0) 0x7f0ec801d7f0 con 0x7f0ed40f6510 2023-11-17T05:29:09.383 INFO:teuthology.orchestra.run.smithi066.stdout: 2023-11-17T05:29:09.383 INFO:teuthology.orchestra.run.smithi066.stdout:{"epoch":1,"fsid":"ed6470e2-8508-11ee-8dbe-212e2dc638e7","modified":"2023-11-17T05:20:13.934038Z","created":"2023-11-17T05:20:13.934038Z","min_mon_release":16,"min_mon_release_name":"pacific","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging"],"optional":[]},"mons":[{"rank":0,"name":"smithi040","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:3300","nonce":0},{"type":"v1","addr":"172.21.15.40:6789","nonce":0}]},"addr":"172.21.15.40:6789/0","public_addr":"172.21.15.40:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2023-11-17T05:29:09.384 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:09.383+0000 7f0edab0c700 1 -- 172.21.15.66:0/409860994 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f0ec402ffb0 msgr2=0x7f0ec4032470 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:29:09.384 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:09.383+0000 7f0edab0c700 1 --2- 172.21.15.66:0/409860994 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f0ec402ffb0 0x7f0ec4032470 secure :-1 s=READY pgs=103 cs=0 l=1 rev1=1 rx=0x7f0ec00096a0 tx=0x7f0ec0006b40).stop 2023-11-17T05:29:09.384 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:09.383+0000 7f0edab0c700 1 -- 172.21.15.66:0/409860994 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f0ed40f6510 msgr2=0x7f0ed406b9a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:29:09.384 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:09.383+0000 7f0edab0c700 1 --2- 172.21.15.66:0/409860994 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f0ed40f6510 0x7f0ed406b9a0 secure :-1 s=READY pgs=276 cs=0 l=1 rev1=1 rx=0x7f0ec8013510 tx=0x7f0ec800b040).stop 2023-11-17T05:29:09.384 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:09.383+0000 7f0edab0c700 1 -- 172.21.15.66:0/409860994 shutdown_connections 2023-11-17T05:29:09.385 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:09.383+0000 7f0edab0c700 1 --2- 172.21.15.66:0/409860994 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f0ec402ffb0 0x7f0ec4032470 unknown :-1 s=CLOSED pgs=103 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:29:09.385 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:09.383+0000 7f0edab0c700 1 --2- 172.21.15.66:0/409860994 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f0ed40f6510 0x7f0ed406b9a0 unknown :-1 s=CLOSED pgs=276 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:29:09.385 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:09.383+0000 7f0edab0c700 1 -- 172.21.15.66:0/409860994 >> 172.21.15.66:0/409860994 conn(0x7f0ed40f1c70 msgr2=0x7f0ed40f2760 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:29:09.385 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:09.383+0000 7f0edab0c700 1 -- 172.21.15.66:0/409860994 shutdown_connections 2023-11-17T05:29:09.385 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:09.383+0000 7f0edab0c700 1 -- 172.21.15.66:0/409860994 wait complete. 2023-11-17T05:29:09.385 INFO:teuthology.orchestra.run.smithi066.stderr:dumped monmap epoch 1 2023-11-17T05:29:09.989 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:29:09 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:29:09.382685+0000 mon.smithi040 (mon.0) 234 : audit [DBG] from='client.? 172.21.15.66:0/409860994' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-11-17T05:29:10.779 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2023-11-17T05:29:10.779 DEBUG:teuthology.orchestra.run.smithi066:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.4 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ed6470e2-8508-11ee-8dbe-212e2dc638e7 -- ceph mon dump -f json 2023-11-17T05:29:10.989 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:29:10 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:29:09.235173+0000 mgr.smithi040.ogcprl (mgr.14164) 247 : cluster [DBG] pgmap v233: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:29:11.973 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:11.971+0000 7fef00e78700 1 -- 172.21.15.66:0/604259583 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7feefc0f5410 msgr2=0x7feefc0f5830 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:29:11.973 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:11.971+0000 7fef00e78700 1 --2- 172.21.15.66:0/604259583 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7feefc0f5410 0x7feefc0f5830 secure :-1 s=READY pgs=277 cs=0 l=1 rev1=1 rx=0x7feeec009fb0 tx=0x7feeec009b30).stop 2023-11-17T05:29:11.974 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:11.971+0000 7fef00e78700 1 -- 172.21.15.66:0/604259583 shutdown_connections 2023-11-17T05:29:11.974 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:11.971+0000 7fef00e78700 1 --2- 172.21.15.66:0/604259583 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7feefc0f5410 0x7feefc0f5830 unknown :-1 s=CLOSED pgs=277 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:29:11.974 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:11.971+0000 7fef00e78700 1 -- 172.21.15.66:0/604259583 >> 172.21.15.66:0/604259583 conn(0x7feefc0f0b70 msgr2=0x7feefc0f2f90 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:29:11.974 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:11.971+0000 7fef00e78700 1 -- 172.21.15.66:0/604259583 shutdown_connections 2023-11-17T05:29:11.974 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:11.971+0000 7fef00e78700 1 -- 172.21.15.66:0/604259583 wait complete. 2023-11-17T05:29:11.974 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:11.972+0000 7fef00e78700 1 Processor -- start 2023-11-17T05:29:11.974 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:11.972+0000 7fef00e78700 1 -- start start 2023-11-17T05:29:11.974 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:11.972+0000 7fef00e78700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7feefc0f5410 0x7feefc109430 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:29:11.974 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:11.972+0000 7fef00e78700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7feefc060240 con 0x7feefc0f5410 2023-11-17T05:29:11.976 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:11.973+0000 7feefa59c700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7feefc0f5410 0x7feefc109430 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:29:11.976 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:11.973+0000 7feefa59c700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7feefc0f5410 0x7feefc109430 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.66:52388/0 (socket says 172.21.15.66:52388) 2023-11-17T05:29:11.976 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:11.973+0000 7feefa59c700 1 -- 172.21.15.66:0/743240231 learned_addr learned my addr 172.21.15.66:0/743240231 (peer_addr_for_me v2:172.21.15.66:0/0) 2023-11-17T05:29:11.976 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:11.973+0000 7feefa59c700 1 -- 172.21.15.66:0/743240231 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7feeec005040 con 0x7feefc0f5410 2023-11-17T05:29:11.976 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:11.973+0000 7feefa59c700 1 --2- 172.21.15.66:0/743240231 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7feefc0f5410 0x7feefc109430 secure :-1 s=READY pgs=278 cs=0 l=1 rev1=1 rx=0x7feeec013510 tx=0x7feeec00b040).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:29:11.976 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:11.973+0000 7feeeaffd700 1 -- 172.21.15.66:0/743240231 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7feeec009320 con 0x7feefc0f5410 2023-11-17T05:29:11.976 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:11.973+0000 7fef00e78700 1 -- 172.21.15.66:0/743240231 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7feefc060440 con 0x7feefc0f5410 2023-11-17T05:29:11.976 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:11.973+0000 7feeeaffd700 1 -- 172.21.15.66:0/743240231 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(25 keys) v1 ==== 1074+0+0 (secure 0 0 0) 0x7feeec013f00 con 0x7feefc0f5410 2023-11-17T05:29:11.976 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:11.973+0000 7feeeaffd700 1 -- 172.21.15.66:0/743240231 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7feeec014d90 con 0x7feefc0f5410 2023-11-17T05:29:11.976 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:11.973+0000 7fef00e78700 1 -- 172.21.15.66:0/743240231 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7feefc109ae0 con 0x7feefc0f5410 2023-11-17T05:29:11.976 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:11.974+0000 7feeeaffd700 1 -- 172.21.15.66:0/743240231 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 17) v1 ==== 36269+0+0 (secure 0 0 0) 0x7feeec0095f0 con 0x7feefc0f5410 2023-11-17T05:29:11.976 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:11.974+0000 7fef00e78700 1 -- 172.21.15.66:0/743240231 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7feefc045ad0 con 0x7feefc0f5410 2023-11-17T05:29:11.977 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:11.974+0000 7feeeaffd700 1 --2- 172.21.15.66:0/743240231 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7feee402ff60 0x7feee4032420 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:29:11.977 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:11.974+0000 7feeeaffd700 1 -- 172.21.15.66:0/743240231 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(5..5 src has 1..5) v4 ==== 1270+0+0 (secure 0 0 0) 0x7feeec0410c0 con 0x7feefc0f5410 2023-11-17T05:29:11.977 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:11.975+0000 7feef9d9b700 1 --2- 172.21.15.66:0/743240231 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7feee402ff60 0x7feee4032420 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:29:11.977 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:11.975+0000 7feef9d9b700 1 --2- 172.21.15.66:0/743240231 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7feee402ff60 0x7feee4032420 secure :-1 s=READY pgs=104 cs=0 l=1 rev1=1 rx=0x7feef00096a0 tx=0x7feef0006b40).ready entity=mgr.14164 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:29:11.980 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:11.979+0000 7feeeaffd700 1 -- 172.21.15.66:0/743240231 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+152212 (secure 0 0 0) 0x7feeec014420 con 0x7feefc0f5410 2023-11-17T05:29:11.988 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:29:11 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:29:11.315858+0000 mon.smithi040 (mon.0) 235 : audit [INF] from='mgr.14164 172.21.15.40:0/2203032451' entity='mgr.smithi040.ogcprl' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi040.ogcprl/mirror_snapshot_schedule"}]: dispatch 2023-11-17T05:29:11.989 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:29:11 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:29:11.316680+0000 mon.smithi040 (mon.0) 236 : audit [INF] from='mgr.14164 172.21.15.40:0/2203032451' entity='mgr.smithi040.ogcprl' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi040.ogcprl/trash_purge_schedule"}]: dispatch 2023-11-17T05:29:12.202 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:12.201+0000 7fef00e78700 1 -- 172.21.15.66:0/743240231 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "mon dump", "format": "json"} v 0) v1 -- 0x7feefc0fae60 con 0x7feefc0f5410 2023-11-17T05:29:12.203 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:12.202+0000 7feeeaffd700 1 -- 172.21.15.66:0/743240231 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mon_command_ack([{"prefix": "mon dump", "format": "json"}]=0 dumped monmap epoch 1 v1) v1 ==== 95+0+690 (secure 0 0 0) 0x7feeec010230 con 0x7feefc0f5410 2023-11-17T05:29:12.203 INFO:teuthology.orchestra.run.smithi066.stdout: 2023-11-17T05:29:12.203 INFO:teuthology.orchestra.run.smithi066.stdout:{"epoch":1,"fsid":"ed6470e2-8508-11ee-8dbe-212e2dc638e7","modified":"2023-11-17T05:20:13.934038Z","created":"2023-11-17T05:20:13.934038Z","min_mon_release":16,"min_mon_release_name":"pacific","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging"],"optional":[]},"mons":[{"rank":0,"name":"smithi040","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:3300","nonce":0},{"type":"v1","addr":"172.21.15.40:6789","nonce":0}]},"addr":"172.21.15.40:6789/0","public_addr":"172.21.15.40:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2023-11-17T05:29:12.205 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:12.203+0000 7fef00e78700 1 -- 172.21.15.66:0/743240231 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7feee402ff60 msgr2=0x7feee4032420 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:29:12.206 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:12.203+0000 7fef00e78700 1 --2- 172.21.15.66:0/743240231 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7feee402ff60 0x7feee4032420 secure :-1 s=READY pgs=104 cs=0 l=1 rev1=1 rx=0x7feef00096a0 tx=0x7feef0006b40).stop 2023-11-17T05:29:12.206 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:12.203+0000 7fef00e78700 1 -- 172.21.15.66:0/743240231 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7feefc0f5410 msgr2=0x7feefc109430 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:29:12.206 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:12.203+0000 7fef00e78700 1 --2- 172.21.15.66:0/743240231 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7feefc0f5410 0x7feefc109430 secure :-1 s=READY pgs=278 cs=0 l=1 rev1=1 rx=0x7feeec013510 tx=0x7feeec00b040).stop 2023-11-17T05:29:12.206 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:12.203+0000 7fef00e78700 1 -- 172.21.15.66:0/743240231 shutdown_connections 2023-11-17T05:29:12.206 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:12.203+0000 7fef00e78700 1 --2- 172.21.15.66:0/743240231 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7feee402ff60 0x7feee4032420 unknown :-1 s=CLOSED pgs=104 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:29:12.206 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:12.203+0000 7fef00e78700 1 --2- 172.21.15.66:0/743240231 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7feefc0f5410 0x7feefc109430 unknown :-1 s=CLOSED pgs=278 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:29:12.206 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:12.203+0000 7fef00e78700 1 -- 172.21.15.66:0/743240231 >> 172.21.15.66:0/743240231 conn(0x7feefc0f0b70 msgr2=0x7feefc0f1660 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:29:12.206 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:12.203+0000 7fef00e78700 1 -- 172.21.15.66:0/743240231 shutdown_connections 2023-11-17T05:29:12.206 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:12.203+0000 7fef00e78700 1 -- 172.21.15.66:0/743240231 wait complete. 2023-11-17T05:29:12.206 INFO:teuthology.orchestra.run.smithi066.stderr:dumped monmap epoch 1 2023-11-17T05:29:12.989 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:29:12 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:29:11.235424+0000 mgr.smithi040.ogcprl (mgr.14164) 248 : cluster [DBG] pgmap v234: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:29:12.989 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:29:12 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:29:12.203110+0000 mon.smithi040 (mon.0) 237 : audit [DBG] from='client.? 172.21.15.66:0/743240231' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-11-17T05:29:13.633 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2023-11-17T05:29:13.633 DEBUG:teuthology.orchestra.run.smithi066:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.4 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ed6470e2-8508-11ee-8dbe-212e2dc638e7 -- ceph mon dump -f json 2023-11-17T05:29:14.822 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:14.821+0000 7f6b8b59e700 1 -- 172.21.15.66:0/3444500117 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f6b8c0f51a0 msgr2=0x7f6b8c0f55c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:29:14.822 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:14.821+0000 7f6b8b59e700 1 --2- 172.21.15.66:0/3444500117 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f6b8c0f51a0 0x7f6b8c0f55c0 secure :-1 s=READY pgs=279 cs=0 l=1 rev1=1 rx=0x7f6b7c009fb0 tx=0x7f6b7c009b30).stop 2023-11-17T05:29:14.823 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:14.821+0000 7f6b8b59e700 1 -- 172.21.15.66:0/3444500117 shutdown_connections 2023-11-17T05:29:14.823 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:14.821+0000 7f6b8b59e700 1 --2- 172.21.15.66:0/3444500117 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f6b8c0f51a0 0x7f6b8c0f55c0 unknown :-1 s=CLOSED pgs=279 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:29:14.823 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:14.821+0000 7f6b8b59e700 1 -- 172.21.15.66:0/3444500117 >> 172.21.15.66:0/3444500117 conn(0x7f6b8c0f09a0 msgr2=0x7f6b8c0f2dc0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:29:14.823 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:14.821+0000 7f6b8b59e700 1 -- 172.21.15.66:0/3444500117 shutdown_connections 2023-11-17T05:29:14.823 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:14.821+0000 7f6b8b59e700 1 -- 172.21.15.66:0/3444500117 wait complete. 2023-11-17T05:29:14.824 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:14.822+0000 7f6b8b59e700 1 Processor -- start 2023-11-17T05:29:14.824 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:14.822+0000 7f6b8b59e700 1 -- start start 2023-11-17T05:29:14.824 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:14.822+0000 7f6b8b59e700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f6b8c0f51a0 0x7f6b8c1064a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:29:14.824 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:14.822+0000 7f6b8b59e700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f6b8c106990 con 0x7f6b8c0f51a0 2023-11-17T05:29:14.825 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:14.822+0000 7f6b8a59c700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f6b8c0f51a0 0x7f6b8c1064a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:29:14.825 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:14.822+0000 7f6b8a59c700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f6b8c0f51a0 0x7f6b8c1064a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.66:42360/0 (socket says 172.21.15.66:42360) 2023-11-17T05:29:14.825 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:14.822+0000 7f6b8a59c700 1 -- 172.21.15.66:0/3520479003 learned_addr learned my addr 172.21.15.66:0/3520479003 (peer_addr_for_me v2:172.21.15.66:0/0) 2023-11-17T05:29:14.825 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:14.822+0000 7f6b8a59c700 1 -- 172.21.15.66:0/3520479003 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f6b7c005040 con 0x7f6b8c0f51a0 2023-11-17T05:29:14.825 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:14.822+0000 7f6b8a59c700 1 --2- 172.21.15.66:0/3520479003 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f6b8c0f51a0 0x7f6b8c1064a0 secure :-1 s=READY pgs=280 cs=0 l=1 rev1=1 rx=0x7f6b7c013510 tx=0x7f6b7c00b040).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:29:14.825 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:14.822+0000 7f6b7affd700 1 -- 172.21.15.66:0/3520479003 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f6b7c009320 con 0x7f6b8c0f51a0 2023-11-17T05:29:14.825 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:14.822+0000 7f6b8b59e700 1 -- 172.21.15.66:0/3520479003 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f6b8c106b90 con 0x7f6b8c0f51a0 2023-11-17T05:29:14.825 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:14.823+0000 7f6b7affd700 1 -- 172.21.15.66:0/3520479003 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(25 keys) v1 ==== 1074+0+0 (secure 0 0 0) 0x7f6b7c013f00 con 0x7f6b8c0f51a0 2023-11-17T05:29:14.825 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:14.823+0000 7f6b7affd700 1 -- 172.21.15.66:0/3520479003 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f6b7c014dc0 con 0x7f6b8c0f51a0 2023-11-17T05:29:14.825 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:14.823+0000 7f6b8b59e700 1 -- 172.21.15.66:0/3520479003 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f6b8c107020 con 0x7f6b8c0f51a0 2023-11-17T05:29:14.826 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:14.823+0000 7f6b7affd700 1 -- 172.21.15.66:0/3520479003 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 17) v1 ==== 36269+0+0 (secure 0 0 0) 0x7f6b7c014420 con 0x7f6b8c0f51a0 2023-11-17T05:29:14.826 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:14.823+0000 7f6b8b59e700 1 -- 172.21.15.66:0/3520479003 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f6b8c045ad0 con 0x7f6b8c0f51a0 2023-11-17T05:29:14.826 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:14.824+0000 7f6b7affd700 1 --2- 172.21.15.66:0/3520479003 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f6b7402ffb0 0x7f6b74032470 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:29:14.826 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:14.824+0000 7f6b7affd700 1 -- 172.21.15.66:0/3520479003 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(5..5 src has 1..5) v4 ==== 1270+0+0 (secure 0 0 0) 0x7f6b7c040fe0 con 0x7f6b8c0f51a0 2023-11-17T05:29:14.826 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:14.824+0000 7f6b89d9b700 1 --2- 172.21.15.66:0/3520479003 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f6b7402ffb0 0x7f6b74032470 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:29:14.826 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:14.824+0000 7f6b89d9b700 1 --2- 172.21.15.66:0/3520479003 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f6b7402ffb0 0x7f6b74032470 secure :-1 s=READY pgs=105 cs=0 l=1 rev1=1 rx=0x7f6b800096a0 tx=0x7f6b80006b40).ready entity=mgr.14164 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:29:14.830 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:14.829+0000 7f6b7affd700 1 -- 172.21.15.66:0/3520479003 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+152212 (secure 0 0 0) 0x7f6b7c06d2b0 con 0x7f6b8c0f51a0 2023-11-17T05:29:14.989 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:29:14 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:29:13.235721+0000 mgr.smithi040.ogcprl (mgr.14164) 249 : cluster [DBG] pgmap v235: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:29:15.050 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:15.049+0000 7f6b8b59e700 1 -- 172.21.15.66:0/3520479003 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "mon dump", "format": "json"} v 0) v1 -- 0x7f6b8c107ed0 con 0x7f6b8c0f51a0 2023-11-17T05:29:15.050 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:15.049+0000 7f6b7affd700 1 -- 172.21.15.66:0/3520479003 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mon_command_ack([{"prefix": "mon dump", "format": "json"}]=0 dumped monmap epoch 1 v1) v1 ==== 95+0+690 (secure 0 0 0) 0x7f6b7c0095f0 con 0x7f6b8c0f51a0 2023-11-17T05:29:15.051 INFO:teuthology.orchestra.run.smithi066.stdout: 2023-11-17T05:29:15.051 INFO:teuthology.orchestra.run.smithi066.stdout:{"epoch":1,"fsid":"ed6470e2-8508-11ee-8dbe-212e2dc638e7","modified":"2023-11-17T05:20:13.934038Z","created":"2023-11-17T05:20:13.934038Z","min_mon_release":16,"min_mon_release_name":"pacific","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging"],"optional":[]},"mons":[{"rank":0,"name":"smithi040","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:3300","nonce":0},{"type":"v1","addr":"172.21.15.40:6789","nonce":0}]},"addr":"172.21.15.40:6789/0","public_addr":"172.21.15.40:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2023-11-17T05:29:15.052 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:15.051+0000 7f6b8b59e700 1 -- 172.21.15.66:0/3520479003 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f6b7402ffb0 msgr2=0x7f6b74032470 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:29:15.052 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:15.051+0000 7f6b8b59e700 1 --2- 172.21.15.66:0/3520479003 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f6b7402ffb0 0x7f6b74032470 secure :-1 s=READY pgs=105 cs=0 l=1 rev1=1 rx=0x7f6b800096a0 tx=0x7f6b80006b40).stop 2023-11-17T05:29:15.052 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:15.051+0000 7f6b8b59e700 1 -- 172.21.15.66:0/3520479003 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f6b8c0f51a0 msgr2=0x7f6b8c1064a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:29:15.052 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:15.051+0000 7f6b8b59e700 1 --2- 172.21.15.66:0/3520479003 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f6b8c0f51a0 0x7f6b8c1064a0 secure :-1 s=READY pgs=280 cs=0 l=1 rev1=1 rx=0x7f6b7c013510 tx=0x7f6b7c00b040).stop 2023-11-17T05:29:15.052 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:15.051+0000 7f6b8b59e700 1 -- 172.21.15.66:0/3520479003 shutdown_connections 2023-11-17T05:29:15.052 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:15.051+0000 7f6b8b59e700 1 --2- 172.21.15.66:0/3520479003 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f6b7402ffb0 0x7f6b74032470 unknown :-1 s=CLOSED pgs=105 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:29:15.052 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:15.051+0000 7f6b8b59e700 1 --2- 172.21.15.66:0/3520479003 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f6b8c0f51a0 0x7f6b8c1064a0 unknown :-1 s=CLOSED pgs=280 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:29:15.052 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:15.051+0000 7f6b8b59e700 1 -- 172.21.15.66:0/3520479003 >> 172.21.15.66:0/3520479003 conn(0x7f6b8c0f09a0 msgr2=0x7f6b8c05fde0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:29:15.053 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:15.051+0000 7f6b8b59e700 1 -- 172.21.15.66:0/3520479003 shutdown_connections 2023-11-17T05:29:15.053 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:15.051+0000 7f6b8b59e700 1 -- 172.21.15.66:0/3520479003 wait complete. 2023-11-17T05:29:15.053 INFO:teuthology.orchestra.run.smithi066.stderr:dumped monmap epoch 1 2023-11-17T05:29:15.989 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:29:15 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:29:15.050931+0000 mon.smithi040 (mon.0) 238 : audit [DBG] from='client.? 172.21.15.66:0/3520479003' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-11-17T05:29:16.457 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2023-11-17T05:29:16.458 DEBUG:teuthology.orchestra.run.smithi066:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.4 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ed6470e2-8508-11ee-8dbe-212e2dc638e7 -- ceph mon dump -f json 2023-11-17T05:29:16.990 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:29:16 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:29:15.236053+0000 mgr.smithi040.ogcprl (mgr.14164) 250 : cluster [DBG] pgmap v236: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:29:17.648 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:17.645+0000 7f31142cc700 1 -- 172.21.15.66:0/2652746295 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f310c0f6510 msgr2=0x7f310c0f6930 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:29:17.648 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:17.645+0000 7f31142cc700 1 --2- 172.21.15.66:0/2652746295 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f310c0f6510 0x7f310c0f6930 secure :-1 s=READY pgs=281 cs=0 l=1 rev1=1 rx=0x7f30fc009fb0 tx=0x7f30fc009b30).stop 2023-11-17T05:29:17.648 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:17.645+0000 7f31142cc700 1 -- 172.21.15.66:0/2652746295 shutdown_connections 2023-11-17T05:29:17.648 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:17.645+0000 7f31142cc700 1 --2- 172.21.15.66:0/2652746295 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f310c0f6510 0x7f310c0f6930 unknown :-1 s=CLOSED pgs=281 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:29:17.648 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:17.645+0000 7f31142cc700 1 -- 172.21.15.66:0/2652746295 >> 172.21.15.66:0/2652746295 conn(0x7f310c0f1c70 msgr2=0x7f310c0f4090 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:29:17.648 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:17.645+0000 7f31142cc700 1 -- 172.21.15.66:0/2652746295 shutdown_connections 2023-11-17T05:29:17.649 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:17.645+0000 7f31142cc700 1 -- 172.21.15.66:0/2652746295 wait complete. 2023-11-17T05:29:17.649 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:17.646+0000 7f31142cc700 1 Processor -- start 2023-11-17T05:29:17.649 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:17.646+0000 7f31142cc700 1 -- start start 2023-11-17T05:29:17.649 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:17.646+0000 7f31142cc700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f310c0f6510 0x7f310c106730 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:29:17.649 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:17.646+0000 7f31142cc700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f310c106c20 con 0x7f310c0f6510 2023-11-17T05:29:17.649 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:17.646+0000 7f3112068700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f310c0f6510 0x7f310c106730 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:29:17.649 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:17.646+0000 7f3112068700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f310c0f6510 0x7f310c106730 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.66:42378/0 (socket says 172.21.15.66:42378) 2023-11-17T05:29:17.649 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:17.646+0000 7f3112068700 1 -- 172.21.15.66:0/3057898644 learned_addr learned my addr 172.21.15.66:0/3057898644 (peer_addr_for_me v2:172.21.15.66:0/0) 2023-11-17T05:29:17.650 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:17.646+0000 7f3112068700 1 -- 172.21.15.66:0/3057898644 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f30fc005040 con 0x7f310c0f6510 2023-11-17T05:29:17.650 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:17.647+0000 7f3112068700 1 --2- 172.21.15.66:0/3057898644 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f310c0f6510 0x7f310c106730 secure :-1 s=READY pgs=282 cs=0 l=1 rev1=1 rx=0x7f30fc013510 tx=0x7f30fc00b040).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:29:17.650 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:17.647+0000 7f3102ffd700 1 -- 172.21.15.66:0/3057898644 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f30fc009320 con 0x7f310c0f6510 2023-11-17T05:29:17.650 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:17.647+0000 7f31142cc700 1 -- 172.21.15.66:0/3057898644 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f310c106e20 con 0x7f310c0f6510 2023-11-17T05:29:17.650 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:17.647+0000 7f3102ffd700 1 -- 172.21.15.66:0/3057898644 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(25 keys) v1 ==== 1074+0+0 (secure 0 0 0) 0x7f30fc013f00 con 0x7f310c0f6510 2023-11-17T05:29:17.650 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:17.647+0000 7f3102ffd700 1 -- 172.21.15.66:0/3057898644 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f30fc014d90 con 0x7f310c0f6510 2023-11-17T05:29:17.650 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:17.647+0000 7f31142cc700 1 -- 172.21.15.66:0/3057898644 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f310c1072b0 con 0x7f310c0f6510 2023-11-17T05:29:17.651 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:17.648+0000 7f3102ffd700 1 -- 172.21.15.66:0/3057898644 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 17) v1 ==== 36269+0+0 (secure 0 0 0) 0x7f30fc0263a0 con 0x7f310c0f6510 2023-11-17T05:29:17.651 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:17.648+0000 7f3102ffd700 1 --2- 172.21.15.66:0/3057898644 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f30f802ff60 0x7f30f8032420 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:29:17.651 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:17.648+0000 7f31142cc700 1 -- 172.21.15.66:0/3057898644 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f310c045ad0 con 0x7f310c0f6510 2023-11-17T05:29:17.651 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:17.648+0000 7f3102ffd700 1 -- 172.21.15.66:0/3057898644 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(5..5 src has 1..5) v4 ==== 1270+0+0 (secure 0 0 0) 0x7f30fc041000 con 0x7f310c0f6510 2023-11-17T05:29:17.651 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:17.648+0000 7f3111867700 1 --2- 172.21.15.66:0/3057898644 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f30f802ff60 0x7f30f8032420 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:29:17.651 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:17.648+0000 7f3111867700 1 --2- 172.21.15.66:0/3057898644 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f30f802ff60 0x7f30f8032420 secure :-1 s=READY pgs=106 cs=0 l=1 rev1=1 rx=0x7f31080096a0 tx=0x7f3108006b40).ready entity=mgr.14164 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:29:17.655 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:17.653+0000 7f3102ffd700 1 -- 172.21.15.66:0/3057898644 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+152212 (secure 0 0 0) 0x7f30fc012920 con 0x7f310c0f6510 2023-11-17T05:29:17.872 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:17.871+0000 7f31142cc700 1 -- 172.21.15.66:0/3057898644 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "mon dump", "format": "json"} v 0) v1 -- 0x7f310c0fc050 con 0x7f310c0f6510 2023-11-17T05:29:17.872 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:17.871+0000 7f3102ffd700 1 -- 172.21.15.66:0/3057898644 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mon_command_ack([{"prefix": "mon dump", "format": "json"}]=0 dumped monmap epoch 1 v1) v1 ==== 95+0+690 (secure 0 0 0) 0x7f30fc00f340 con 0x7f310c0f6510 2023-11-17T05:29:17.873 INFO:teuthology.orchestra.run.smithi066.stdout: 2023-11-17T05:29:17.873 INFO:teuthology.orchestra.run.smithi066.stdout:{"epoch":1,"fsid":"ed6470e2-8508-11ee-8dbe-212e2dc638e7","modified":"2023-11-17T05:20:13.934038Z","created":"2023-11-17T05:20:13.934038Z","min_mon_release":16,"min_mon_release_name":"pacific","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging"],"optional":[]},"mons":[{"rank":0,"name":"smithi040","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:3300","nonce":0},{"type":"v1","addr":"172.21.15.40:6789","nonce":0}]},"addr":"172.21.15.40:6789/0","public_addr":"172.21.15.40:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2023-11-17T05:29:17.874 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:17.873+0000 7f31142cc700 1 -- 172.21.15.66:0/3057898644 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f30f802ff60 msgr2=0x7f30f8032420 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:29:17.874 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:17.873+0000 7f31142cc700 1 --2- 172.21.15.66:0/3057898644 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f30f802ff60 0x7f30f8032420 secure :-1 s=READY pgs=106 cs=0 l=1 rev1=1 rx=0x7f31080096a0 tx=0x7f3108006b40).stop 2023-11-17T05:29:17.874 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:17.873+0000 7f31142cc700 1 -- 172.21.15.66:0/3057898644 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f310c0f6510 msgr2=0x7f310c106730 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:29:17.875 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:17.873+0000 7f31142cc700 1 --2- 172.21.15.66:0/3057898644 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f310c0f6510 0x7f310c106730 secure :-1 s=READY pgs=282 cs=0 l=1 rev1=1 rx=0x7f30fc013510 tx=0x7f30fc00b040).stop 2023-11-17T05:29:17.875 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:17.873+0000 7f31142cc700 1 -- 172.21.15.66:0/3057898644 shutdown_connections 2023-11-17T05:29:17.875 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:17.873+0000 7f31142cc700 1 --2- 172.21.15.66:0/3057898644 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f30f802ff60 0x7f30f8032420 unknown :-1 s=CLOSED pgs=106 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:29:17.875 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:17.873+0000 7f31142cc700 1 --2- 172.21.15.66:0/3057898644 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f310c0f6510 0x7f310c106730 unknown :-1 s=CLOSED pgs=282 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:29:17.875 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:17.873+0000 7f31142cc700 1 -- 172.21.15.66:0/3057898644 >> 172.21.15.66:0/3057898644 conn(0x7f310c0f1c70 msgr2=0x7f310c0f2760 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:29:17.875 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:17.873+0000 7f31142cc700 1 -- 172.21.15.66:0/3057898644 shutdown_connections 2023-11-17T05:29:17.875 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:17.873+0000 7f31142cc700 1 -- 172.21.15.66:0/3057898644 wait complete. 2023-11-17T05:29:17.875 INFO:teuthology.orchestra.run.smithi066.stderr:dumped monmap epoch 1 2023-11-17T05:29:17.988 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:29:17 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:29:17.236380+0000 mgr.smithi040.ogcprl (mgr.14164) 251 : cluster [DBG] pgmap v237: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:29:18.989 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:29:18 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:29:17.873069+0000 mon.smithi040 (mon.0) 239 : audit [DBG] from='client.? 172.21.15.66:0/3057898644' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-11-17T05:29:19.258 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2023-11-17T05:29:19.259 DEBUG:teuthology.orchestra.run.smithi066:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.4 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ed6470e2-8508-11ee-8dbe-212e2dc638e7 -- ceph mon dump -f json 2023-11-17T05:29:20.443 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:20.442+0000 7fcbe93fe700 1 -- 172.21.15.66:0/3721197275 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fcbe40f62c0 msgr2=0x7fcbe40f66e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:29:20.444 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:20.442+0000 7fcbe93fe700 1 --2- 172.21.15.66:0/3721197275 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fcbe40f62c0 0x7fcbe40f66e0 secure :-1 s=READY pgs=283 cs=0 l=1 rev1=1 rx=0x7fcbd4009fb0 tx=0x7fcbd4009b30).stop 2023-11-17T05:29:20.444 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:20.442+0000 7fcbe93fe700 1 -- 172.21.15.66:0/3721197275 shutdown_connections 2023-11-17T05:29:20.444 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:20.442+0000 7fcbe93fe700 1 --2- 172.21.15.66:0/3721197275 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fcbe40f62c0 0x7fcbe40f66e0 unknown :-1 s=CLOSED pgs=283 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:29:20.444 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:20.442+0000 7fcbe93fe700 1 -- 172.21.15.66:0/3721197275 >> 172.21.15.66:0/3721197275 conn(0x7fcbe40f1ac0 msgr2=0x7fcbe40f3ee0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:29:20.444 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:20.442+0000 7fcbe93fe700 1 -- 172.21.15.66:0/3721197275 shutdown_connections 2023-11-17T05:29:20.444 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:20.442+0000 7fcbe93fe700 1 -- 172.21.15.66:0/3721197275 wait complete. 2023-11-17T05:29:20.444 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:20.443+0000 7fcbe93fe700 1 Processor -- start 2023-11-17T05:29:20.445 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:20.443+0000 7fcbe93fe700 1 -- start start 2023-11-17T05:29:20.445 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:20.443+0000 7fcbe93fe700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fcbe40f62c0 0x7fcbe4106520 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:29:20.446 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:20.443+0000 7fcbe93fe700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fcbe4106a10 con 0x7fcbe40f62c0 2023-11-17T05:29:20.446 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:20.443+0000 7fcbe3fff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fcbe40f62c0 0x7fcbe4106520 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:29:20.446 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:20.443+0000 7fcbe3fff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fcbe40f62c0 0x7fcbe4106520 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.66:42402/0 (socket says 172.21.15.66:42402) 2023-11-17T05:29:20.446 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:20.443+0000 7fcbe3fff700 1 -- 172.21.15.66:0/3422175896 learned_addr learned my addr 172.21.15.66:0/3422175896 (peer_addr_for_me v2:172.21.15.66:0/0) 2023-11-17T05:29:20.446 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:20.443+0000 7fcbe3fff700 1 -- 172.21.15.66:0/3422175896 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fcbd4005040 con 0x7fcbe40f62c0 2023-11-17T05:29:20.446 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:20.444+0000 7fcbe3fff700 1 --2- 172.21.15.66:0/3422175896 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fcbe40f62c0 0x7fcbe4106520 secure :-1 s=READY pgs=284 cs=0 l=1 rev1=1 rx=0x7fcbd4013510 tx=0x7fcbd400b040).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:29:20.446 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:20.444+0000 7fcbe0ff9700 1 -- 172.21.15.66:0/3422175896 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7fcbd4009320 con 0x7fcbe40f62c0 2023-11-17T05:29:20.446 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:20.444+0000 7fcbe93fe700 1 -- 172.21.15.66:0/3422175896 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fcbe4106c10 con 0x7fcbe40f62c0 2023-11-17T05:29:20.446 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:20.444+0000 7fcbe93fe700 1 -- 172.21.15.66:0/3422175896 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fcbe41070a0 con 0x7fcbe40f62c0 2023-11-17T05:29:20.446 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:20.444+0000 7fcbe0ff9700 1 -- 172.21.15.66:0/3422175896 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(25 keys) v1 ==== 1074+0+0 (secure 0 0 0) 0x7fcbd4013f00 con 0x7fcbe40f62c0 2023-11-17T05:29:20.446 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:20.444+0000 7fcbe0ff9700 1 -- 172.21.15.66:0/3422175896 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7fcbd4014d90 con 0x7fcbe40f62c0 2023-11-17T05:29:20.447 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:20.445+0000 7fcbe0ff9700 1 -- 172.21.15.66:0/3422175896 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 17) v1 ==== 36269+0+0 (secure 0 0 0) 0x7fcbd4014420 con 0x7fcbe40f62c0 2023-11-17T05:29:20.447 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:20.445+0000 7fcbe93fe700 1 -- 172.21.15.66:0/3422175896 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fcbe4045ad0 con 0x7fcbe40f62c0 2023-11-17T05:29:20.447 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:20.445+0000 7fcbe0ff9700 1 --2- 172.21.15.66:0/3422175896 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fcbd002ff60 0x7fcbd0032420 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:29:20.447 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:20.445+0000 7fcbe0ff9700 1 -- 172.21.15.66:0/3422175896 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(5..5 src has 1..5) v4 ==== 1270+0+0 (secure 0 0 0) 0x7fcbd4041290 con 0x7fcbe40f62c0 2023-11-17T05:29:20.447 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:20.445+0000 7fcbe37fe700 1 --2- 172.21.15.66:0/3422175896 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fcbd002ff60 0x7fcbd0032420 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:29:20.448 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:20.446+0000 7fcbe37fe700 1 --2- 172.21.15.66:0/3422175896 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fcbd002ff60 0x7fcbd0032420 secure :-1 s=READY pgs=107 cs=0 l=1 rev1=1 rx=0x7fcbcc0096a0 tx=0x7fcbcc006b40).ready entity=mgr.14164 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:29:20.452 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:20.450+0000 7fcbe0ff9700 1 -- 172.21.15.66:0/3422175896 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+152212 (secure 0 0 0) 0x7fcbd400fb30 con 0x7fcbe40f62c0 2023-11-17T05:29:20.671 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:20.667+0000 7fcbe93fe700 1 -- 172.21.15.66:0/3422175896 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "mon dump", "format": "json"} v 0) v1 -- 0x7fcbe4107f50 con 0x7fcbe40f62c0 2023-11-17T05:29:20.671 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:20.668+0000 7fcbe0ff9700 1 -- 172.21.15.66:0/3422175896 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mon_command_ack([{"prefix": "mon dump", "format": "json"}]=0 dumped monmap epoch 1 v1) v1 ==== 95+0+690 (secure 0 0 0) 0x7fcbd400fb30 con 0x7fcbe40f62c0 2023-11-17T05:29:20.672 INFO:teuthology.orchestra.run.smithi066.stdout: 2023-11-17T05:29:20.672 INFO:teuthology.orchestra.run.smithi066.stdout:{"epoch":1,"fsid":"ed6470e2-8508-11ee-8dbe-212e2dc638e7","modified":"2023-11-17T05:20:13.934038Z","created":"2023-11-17T05:20:13.934038Z","min_mon_release":16,"min_mon_release_name":"pacific","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging"],"optional":[]},"mons":[{"rank":0,"name":"smithi040","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:3300","nonce":0},{"type":"v1","addr":"172.21.15.40:6789","nonce":0}]},"addr":"172.21.15.40:6789/0","public_addr":"172.21.15.40:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2023-11-17T05:29:20.672 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:20.669+0000 7fcbe93fe700 1 -- 172.21.15.66:0/3422175896 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fcbd002ff60 msgr2=0x7fcbd0032420 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:29:20.673 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:20.669+0000 7fcbe93fe700 1 --2- 172.21.15.66:0/3422175896 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fcbd002ff60 0x7fcbd0032420 secure :-1 s=READY pgs=107 cs=0 l=1 rev1=1 rx=0x7fcbcc0096a0 tx=0x7fcbcc006b40).stop 2023-11-17T05:29:20.673 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:20.669+0000 7fcbe93fe700 1 -- 172.21.15.66:0/3422175896 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fcbe40f62c0 msgr2=0x7fcbe4106520 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:29:20.673 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:20.669+0000 7fcbe93fe700 1 --2- 172.21.15.66:0/3422175896 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fcbe40f62c0 0x7fcbe4106520 secure :-1 s=READY pgs=284 cs=0 l=1 rev1=1 rx=0x7fcbd4013510 tx=0x7fcbd400b040).stop 2023-11-17T05:29:20.673 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:20.669+0000 7fcbe93fe700 1 -- 172.21.15.66:0/3422175896 shutdown_connections 2023-11-17T05:29:20.673 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:20.669+0000 7fcbe93fe700 1 --2- 172.21.15.66:0/3422175896 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fcbd002ff60 0x7fcbd0032420 unknown :-1 s=CLOSED pgs=107 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:29:20.673 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:20.669+0000 7fcbe93fe700 1 --2- 172.21.15.66:0/3422175896 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fcbe40f62c0 0x7fcbe4106520 unknown :-1 s=CLOSED pgs=284 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:29:20.673 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:20.669+0000 7fcbe93fe700 1 -- 172.21.15.66:0/3422175896 >> 172.21.15.66:0/3422175896 conn(0x7fcbe40f1ac0 msgr2=0x7fcbe40f2530 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:29:20.673 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:20.669+0000 7fcbe93fe700 1 -- 172.21.15.66:0/3422175896 shutdown_connections 2023-11-17T05:29:20.673 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:20.669+0000 7fcbe93fe700 1 -- 172.21.15.66:0/3422175896 wait complete. 2023-11-17T05:29:20.673 INFO:teuthology.orchestra.run.smithi066.stderr:dumped monmap epoch 1 2023-11-17T05:29:20.739 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:29:20 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:29:19.236825+0000 mgr.smithi040.ogcprl (mgr.14164) 252 : cluster [DBG] pgmap v238: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:29:21.740 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:29:21 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:29:20.669134+0000 mon.smithi040 (mon.0) 240 : audit [DBG] from='client.? 172.21.15.66:0/3422175896' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-11-17T05:29:22.059 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2023-11-17T05:29:22.059 DEBUG:teuthology.orchestra.run.smithi066:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.4 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ed6470e2-8508-11ee-8dbe-212e2dc638e7 -- ceph mon dump -f json 2023-11-17T05:29:22.739 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:29:22 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:29:21.237179+0000 mgr.smithi040.ogcprl (mgr.14164) 253 : cluster [DBG] pgmap v239: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:29:23.223 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:23.220+0000 7fee876bb700 1 -- 172.21.15.66:0/2252559320 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fee800f5410 msgr2=0x7fee800f5830 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:29:23.223 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:23.220+0000 7fee876bb700 1 --2- 172.21.15.66:0/2252559320 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fee800f5410 0x7fee800f5830 secure :-1 s=READY pgs=285 cs=0 l=1 rev1=1 rx=0x7fee7c009fb0 tx=0x7fee7c009b30).stop 2023-11-17T05:29:23.223 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:23.220+0000 7fee876bb700 1 -- 172.21.15.66:0/2252559320 shutdown_connections 2023-11-17T05:29:23.223 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:23.220+0000 7fee876bb700 1 --2- 172.21.15.66:0/2252559320 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fee800f5410 0x7fee800f5830 unknown :-1 s=CLOSED pgs=285 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:29:23.223 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:23.220+0000 7fee876bb700 1 -- 172.21.15.66:0/2252559320 >> 172.21.15.66:0/2252559320 conn(0x7fee800f0b70 msgr2=0x7fee800f2f90 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:29:23.223 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:23.220+0000 7fee876bb700 1 -- 172.21.15.66:0/2252559320 shutdown_connections 2023-11-17T05:29:23.223 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:23.220+0000 7fee876bb700 1 -- 172.21.15.66:0/2252559320 wait complete. 2023-11-17T05:29:23.223 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:23.220+0000 7fee876bb700 1 Processor -- start 2023-11-17T05:29:23.223 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:23.220+0000 7fee876bb700 1 -- start start 2023-11-17T05:29:23.224 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:23.221+0000 7fee876bb700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fee800f5410 0x7fee800fd3a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:29:23.224 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:23.221+0000 7fee876bb700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fee800fb820 con 0x7fee800f5410 2023-11-17T05:29:23.224 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:23.221+0000 7fee85457700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fee800f5410 0x7fee800fd3a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:29:23.224 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:23.221+0000 7fee85457700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fee800f5410 0x7fee800fd3a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.66:37424/0 (socket says 172.21.15.66:37424) 2023-11-17T05:29:23.224 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:23.221+0000 7fee85457700 1 -- 172.21.15.66:0/3923089777 learned_addr learned my addr 172.21.15.66:0/3923089777 (peer_addr_for_me v2:172.21.15.66:0/0) 2023-11-17T05:29:23.224 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:23.221+0000 7fee85457700 1 -- 172.21.15.66:0/3923089777 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fee7c005040 con 0x7fee800f5410 2023-11-17T05:29:23.224 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:23.221+0000 7fee85457700 1 --2- 172.21.15.66:0/3923089777 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fee800f5410 0x7fee800fd3a0 secure :-1 s=READY pgs=286 cs=0 l=1 rev1=1 rx=0x7fee7c013510 tx=0x7fee7c00b040).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:29:23.224 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:23.221+0000 7fee75ffb700 1 -- 172.21.15.66:0/3923089777 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7fee7c009320 con 0x7fee800f5410 2023-11-17T05:29:23.224 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:23.221+0000 7fee876bb700 1 -- 172.21.15.66:0/3923089777 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fee800fb9c0 con 0x7fee800f5410 2023-11-17T05:29:23.224 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:23.221+0000 7fee75ffb700 1 -- 172.21.15.66:0/3923089777 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(25 keys) v1 ==== 1074+0+0 (secure 0 0 0) 0x7fee7c013f00 con 0x7fee800f5410 2023-11-17T05:29:23.224 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:23.222+0000 7fee75ffb700 1 -- 172.21.15.66:0/3923089777 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7fee7c014d90 con 0x7fee800f5410 2023-11-17T05:29:23.224 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:23.222+0000 7fee876bb700 1 -- 172.21.15.66:0/3923089777 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fee800fbe50 con 0x7fee800f5410 2023-11-17T05:29:23.225 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:23.222+0000 7fee876bb700 1 -- 172.21.15.66:0/3923089777 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fee80059100 con 0x7fee800f5410 2023-11-17T05:29:23.225 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:23.222+0000 7fee75ffb700 1 -- 172.21.15.66:0/3923089777 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 17) v1 ==== 36269+0+0 (secure 0 0 0) 0x7fee7c0095f0 con 0x7fee800f5410 2023-11-17T05:29:23.225 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:23.223+0000 7fee75ffb700 1 --2- 172.21.15.66:0/3923089777 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fee6c02ff60 0x7fee6c032420 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:29:23.225 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:23.223+0000 7fee75ffb700 1 -- 172.21.15.66:0/3923089777 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(5..5 src has 1..5) v4 ==== 1270+0+0 (secure 0 0 0) 0x7fee7c040fe0 con 0x7fee800f5410 2023-11-17T05:29:23.225 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:23.223+0000 7fee84c56700 1 --2- 172.21.15.66:0/3923089777 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fee6c02ff60 0x7fee6c032420 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:29:23.226 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:23.223+0000 7fee84c56700 1 --2- 172.21.15.66:0/3923089777 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fee6c02ff60 0x7fee6c032420 secure :-1 s=READY pgs=108 cs=0 l=1 rev1=1 rx=0x7fee700096a0 tx=0x7fee70006b40).ready entity=mgr.14164 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:29:23.231 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:23.227+0000 7fee75ffb700 1 -- 172.21.15.66:0/3923089777 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+152212 (secure 0 0 0) 0x7fee7c014420 con 0x7fee800f5410 2023-11-17T05:29:23.450 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:23.447+0000 7fee876bb700 1 -- 172.21.15.66:0/3923089777 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "mon dump", "format": "json"} v 0) v1 -- 0x7fee80045ad0 con 0x7fee800f5410 2023-11-17T05:29:23.450 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:23.448+0000 7fee75ffb700 1 -- 172.21.15.66:0/3923089777 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mon_command_ack([{"prefix": "mon dump", "format": "json"}]=0 dumped monmap epoch 1 v1) v1 ==== 95+0+690 (secure 0 0 0) 0x7fee7c010230 con 0x7fee800f5410 2023-11-17T05:29:23.451 INFO:teuthology.orchestra.run.smithi066.stdout: 2023-11-17T05:29:23.451 INFO:teuthology.orchestra.run.smithi066.stdout:{"epoch":1,"fsid":"ed6470e2-8508-11ee-8dbe-212e2dc638e7","modified":"2023-11-17T05:20:13.934038Z","created":"2023-11-17T05:20:13.934038Z","min_mon_release":16,"min_mon_release_name":"pacific","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging"],"optional":[]},"mons":[{"rank":0,"name":"smithi040","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:3300","nonce":0},{"type":"v1","addr":"172.21.15.40:6789","nonce":0}]},"addr":"172.21.15.40:6789/0","public_addr":"172.21.15.40:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2023-11-17T05:29:23.451 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:23.449+0000 7fee876bb700 1 -- 172.21.15.66:0/3923089777 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fee6c02ff60 msgr2=0x7fee6c032420 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:29:23.452 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:23.449+0000 7fee876bb700 1 --2- 172.21.15.66:0/3923089777 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fee6c02ff60 0x7fee6c032420 secure :-1 s=READY pgs=108 cs=0 l=1 rev1=1 rx=0x7fee700096a0 tx=0x7fee70006b40).stop 2023-11-17T05:29:23.452 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:23.449+0000 7fee876bb700 1 -- 172.21.15.66:0/3923089777 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fee800f5410 msgr2=0x7fee800fd3a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:29:23.452 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:23.449+0000 7fee876bb700 1 --2- 172.21.15.66:0/3923089777 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fee800f5410 0x7fee800fd3a0 secure :-1 s=READY pgs=286 cs=0 l=1 rev1=1 rx=0x7fee7c013510 tx=0x7fee7c00b040).stop 2023-11-17T05:29:23.452 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:23.449+0000 7fee876bb700 1 -- 172.21.15.66:0/3923089777 shutdown_connections 2023-11-17T05:29:23.452 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:23.449+0000 7fee876bb700 1 --2- 172.21.15.66:0/3923089777 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fee6c02ff60 0x7fee6c032420 unknown :-1 s=CLOSED pgs=108 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:29:23.452 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:23.449+0000 7fee876bb700 1 --2- 172.21.15.66:0/3923089777 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fee800f5410 0x7fee800fd3a0 unknown :-1 s=CLOSED pgs=286 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:29:23.452 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:23.449+0000 7fee876bb700 1 -- 172.21.15.66:0/3923089777 >> 172.21.15.66:0/3923089777 conn(0x7fee800f0b70 msgr2=0x7fee8005fe60 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:29:23.452 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:23.450+0000 7fee876bb700 1 -- 172.21.15.66:0/3923089777 shutdown_connections 2023-11-17T05:29:23.452 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:23.450+0000 7fee876bb700 1 -- 172.21.15.66:0/3923089777 wait complete. 2023-11-17T05:29:23.452 INFO:teuthology.orchestra.run.smithi066.stderr:dumped monmap epoch 1 2023-11-17T05:29:23.739 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:29:23 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:29:23.449439+0000 mon.smithi040 (mon.0) 241 : audit [DBG] from='client.? 172.21.15.66:0/3923089777' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-11-17T05:29:24.739 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:29:24 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:29:23.237559+0000 mgr.smithi040.ogcprl (mgr.14164) 254 : cluster [DBG] pgmap v240: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:29:24.847 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2023-11-17T05:29:24.847 DEBUG:teuthology.orchestra.run.smithi066:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.4 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ed6470e2-8508-11ee-8dbe-212e2dc638e7 -- ceph mon dump -f json 2023-11-17T05:29:26.058 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:26.057+0000 7ff04ef51700 1 -- 172.21.15.66:0/3663502930 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7ff0480f6510 msgr2=0x7ff0480f6930 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:30:23.789 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:26.057+0000 7ff04ef51700 1 --2- 172.21.15.66:0/3663502930 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7ff0480f6510 0x7ff0480f6930 secure :-1 s=READY pgs=287 cs=0 l=1 rev1=1 rx=0x7ff03c009fb0 tx=0x7ff03c009b30).stop 2023-11-17T05:30:23.789 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:26.057+0000 7ff04ef51700 1 -- 172.21.15.66:0/3663502930 shutdown_connections 2023-11-17T05:30:23.789 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:26.057+0000 7ff04ef51700 1 --2- 172.21.15.66:0/3663502930 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7ff0480f6510 0x7ff0480f6930 unknown :-1 s=CLOSED pgs=287 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:30:23.789 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:26.057+0000 7ff04ef51700 1 -- 172.21.15.66:0/3663502930 >> 172.21.15.66:0/3663502930 conn(0x7ff0480f1c70 msgr2=0x7ff0480f4090 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:30:23.790 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:26.057+0000 7ff04ef51700 1 -- 172.21.15.66:0/3663502930 shutdown_connections 2023-11-17T05:30:23.790 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:26.057+0000 7ff04ef51700 1 -- 172.21.15.66:0/3663502930 wait complete. 2023-11-17T05:30:23.811 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:26.058+0000 7ff04ef51700 1 Processor -- start 2023-11-17T05:30:23.811 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:26.058+0000 7ff04ef51700 1 -- start start 2023-11-17T05:30:23.811 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:26.058+0000 7ff04ef51700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7ff0480f6510 0x7ff04806b9a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:30:23.811 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:26.058+0000 7ff04ef51700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7ff048069e20 con 0x7ff0480f6510 2023-11-17T05:30:23.811 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:26.058+0000 7ff04cced700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7ff0480f6510 0x7ff04806b9a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:30:23.812 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:26.058+0000 7ff04cced700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7ff0480f6510 0x7ff04806b9a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.66:37446/0 (socket says 172.21.15.66:37446) 2023-11-17T05:30:23.812 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:26.058+0000 7ff04cced700 1 -- 172.21.15.66:0/3260486409 learned_addr learned my addr 172.21.15.66:0/3260486409 (peer_addr_for_me v2:172.21.15.66:0/0) 2023-11-17T05:30:23.812 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:26.059+0000 7ff04cced700 1 -- 172.21.15.66:0/3260486409 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7ff03c005040 con 0x7ff0480f6510 2023-11-17T05:30:23.812 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:26.059+0000 7ff04cced700 1 --2- 172.21.15.66:0/3260486409 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7ff0480f6510 0x7ff04806b9a0 secure :-1 s=READY pgs=288 cs=0 l=1 rev1=1 rx=0x7ff03c013510 tx=0x7ff03c00b040).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:30:23.855 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:26.059+0000 7ff0457fa700 1 -- 172.21.15.66:0/3260486409 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7ff03c009320 con 0x7ff0480f6510 2023-11-17T05:30:23.888 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:26.059+0000 7ff04ef51700 1 -- 172.21.15.66:0/3260486409 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7ff048069fc0 con 0x7ff0480f6510 2023-11-17T05:30:23.888 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:26.059+0000 7ff04ef51700 1 -- 172.21.15.66:0/3260486409 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7ff04806a450 con 0x7ff0480f6510 2023-11-17T05:30:23.888 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:26.059+0000 7ff0457fa700 1 -- 172.21.15.66:0/3260486409 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(25 keys) v1 ==== 1074+0+0 (secure 0 0 0) 0x7ff03c013f00 con 0x7ff0480f6510 2023-11-17T05:30:23.889 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:26.059+0000 7ff0457fa700 1 -- 172.21.15.66:0/3260486409 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7ff03c014d60 con 0x7ff0480f6510 2023-11-17T05:30:23.889 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:26.060+0000 7ff0457fa700 1 -- 172.21.15.66:0/3260486409 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 17) v1 ==== 36269+0+0 (secure 0 0 0) 0x7ff03c014420 con 0x7ff0480f6510 2023-11-17T05:30:23.889 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:26.060+0000 7ff04ef51700 1 -- 172.21.15.66:0/3260486409 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7ff028004fa0 con 0x7ff0480f6510 2023-11-17T05:30:23.889 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:26.060+0000 7ff0457fa700 1 --2- 172.21.15.66:0/3260486409 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7ff03802ffb0 0x7ff038032470 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:30:23.889 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:26.060+0000 7ff0457fa700 1 -- 172.21.15.66:0/3260486409 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(5..5 src has 1..5) v4 ==== 1270+0+0 (secure 0 0 0) 0x7ff03c0412e0 con 0x7ff0480f6510 2023-11-17T05:30:23.890 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:26.061+0000 7ff047fff700 1 --2- 172.21.15.66:0/3260486409 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7ff03802ffb0 0x7ff038032470 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:30:23.890 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:26.061+0000 7ff047fff700 1 --2- 172.21.15.66:0/3260486409 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7ff03802ffb0 0x7ff038032470 secure :-1 s=READY pgs=109 cs=0 l=1 rev1=1 rx=0x7ff0340096a0 tx=0x7ff034006b40).ready entity=mgr.14164 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:30:23.890 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:26.065+0000 7ff0457fa700 1 -- 172.21.15.66:0/3260486409 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+152212 (secure 0 0 0) 0x7ff03c012b20 con 0x7ff0480f6510 2023-11-17T05:30:23.890 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:26.289+0000 7ff04ef51700 1 -- 172.21.15.66:0/3260486409 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "mon dump", "format": "json"} v 0) v1 -- 0x7ff028005670 con 0x7ff0480f6510 2023-11-17T05:30:23.890 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:26.290+0000 7ff0457fa700 1 -- 172.21.15.66:0/3260486409 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mon_command_ack([{"prefix": "mon dump", "format": "json"}]=0 dumped monmap epoch 1 v1) v1 ==== 95+0+690 (secure 0 0 0) 0x7ff03c012b20 con 0x7ff0480f6510 2023-11-17T05:30:23.891 INFO:teuthology.orchestra.run.smithi066.stdout: 2023-11-17T05:30:23.891 INFO:teuthology.orchestra.run.smithi066.stdout:{"epoch":1,"fsid":"ed6470e2-8508-11ee-8dbe-212e2dc638e7","modified":"2023-11-17T05:20:13.934038Z","created":"2023-11-17T05:20:13.934038Z","min_mon_release":16,"min_mon_release_name":"pacific","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging"],"optional":[]},"mons":[{"rank":0,"name":"smithi040","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:3300","nonce":0},{"type":"v1","addr":"172.21.15.40:6789","nonce":0}]},"addr":"172.21.15.40:6789/0","public_addr":"172.21.15.40:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2023-11-17T05:30:23.895 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:29:26 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:29:25.237994+0000 mgr.smithi040.ogcprl (mgr.14164) 255 : cluster [DBG] pgmap v241: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:30:23.895 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:29:26 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:29:26.291331+0000 mon.smithi040 (mon.0) 242 : audit [DBG] from='client.? 172.21.15.66:0/3260486409' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-11-17T05:30:23.896 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:29:28 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:29:27.238276+0000 mgr.smithi040.ogcprl (mgr.14164) 256 : cluster [DBG] pgmap v242: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:30:23.896 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:29:30 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:29:29.238625+0000 mgr.smithi040.ogcprl (mgr.14164) 257 : cluster [DBG] pgmap v243: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:30:23.896 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:29:32 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:29:31.238988+0000 mgr.smithi040.ogcprl (mgr.14164) 258 : cluster [DBG] pgmap v244: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:30:23.896 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:29:34 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:29:33.239346+0000 mgr.smithi040.ogcprl (mgr.14164) 259 : cluster [DBG] pgmap v245: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:30:23.896 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:29:36 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:29:35.239754+0000 mgr.smithi040.ogcprl (mgr.14164) 260 : cluster [DBG] pgmap v246: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:30:23.897 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:29:38 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:29:37.240048+0000 mgr.smithi040.ogcprl (mgr.14164) 261 : cluster [DBG] pgmap v247: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:30:23.897 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:29:40 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:29:39.240438+0000 mgr.smithi040.ogcprl (mgr.14164) 262 : cluster [DBG] pgmap v248: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:30:23.897 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:29:42 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:29:41.240826+0000 mgr.smithi040.ogcprl (mgr.14164) 263 : cluster [DBG] pgmap v249: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:30:23.897 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:29:44 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:29:43.241202+0000 mgr.smithi040.ogcprl (mgr.14164) 264 : cluster [DBG] pgmap v250: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:30:23.897 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:29:46 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:29:45.241563+0000 mgr.smithi040.ogcprl (mgr.14164) 265 : cluster [DBG] pgmap v251: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:30:23.897 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:29:48 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:29:47.241958+0000 mgr.smithi040.ogcprl (mgr.14164) 266 : cluster [DBG] pgmap v252: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:30:23.897 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:29:50 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:29:49.242256+0000 mgr.smithi040.ogcprl (mgr.14164) 267 : cluster [DBG] pgmap v253: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:30:23.897 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:29:52 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:29:51.242613+0000 mgr.smithi040.ogcprl (mgr.14164) 268 : cluster [DBG] pgmap v254: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:30:23.897 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:29:54 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:29:53.242960+0000 mgr.smithi040.ogcprl (mgr.14164) 269 : cluster [DBG] pgmap v255: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:30:23.897 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:29:56 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:29:55.243336+0000 mgr.smithi040.ogcprl (mgr.14164) 270 : cluster [DBG] pgmap v256: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:30:23.898 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:29:58 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:29:57.243735+0000 mgr.smithi040.ogcprl (mgr.14164) 271 : cluster [DBG] pgmap v257: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:30:23.898 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:30:00 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:29:59.244094+0000 mgr.smithi040.ogcprl (mgr.14164) 272 : cluster [DBG] pgmap v258: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:30:23.898 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:30:02 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:30:01.244420+0000 mgr.smithi040.ogcprl (mgr.14164) 273 : cluster [DBG] pgmap v259: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:30:23.898 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:30:04 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:30:03.244808+0000 mgr.smithi040.ogcprl (mgr.14164) 274 : cluster [DBG] pgmap v260: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:30:23.898 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:30:06 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:30:05.245216+0000 mgr.smithi040.ogcprl (mgr.14164) 275 : cluster [DBG] pgmap v261: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:30:23.898 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:30:08 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:30:07.245553+0000 mgr.smithi040.ogcprl (mgr.14164) 276 : cluster [DBG] pgmap v262: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:30:23.898 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:30:10 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:30:09.245901+0000 mgr.smithi040.ogcprl (mgr.14164) 277 : cluster [DBG] pgmap v263: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:30:23.898 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:30:11 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:30:11.316898+0000 mon.smithi040 (mon.0) 243 : audit [INF] from='mgr.14164 172.21.15.40:0/2203032451' entity='mgr.smithi040.ogcprl' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi040.ogcprl/mirror_snapshot_schedule"}]: dispatch 2023-11-17T05:30:23.898 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:30:11 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:30:11.317671+0000 mon.smithi040 (mon.0) 244 : audit [INF] from='mgr.14164 172.21.15.40:0/2203032451' entity='mgr.smithi040.ogcprl' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi040.ogcprl/trash_purge_schedule"}]: dispatch 2023-11-17T05:30:23.898 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:30:12 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:30:11.246194+0000 mgr.smithi040.ogcprl (mgr.14164) 278 : cluster [DBG] pgmap v264: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:30:23.898 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:30:14 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:30:13.246462+0000 mgr.smithi040.ogcprl (mgr.14164) 279 : cluster [DBG] pgmap v265: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:30:23.898 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:30:16 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:30:15.246831+0000 mgr.smithi040.ogcprl (mgr.14164) 280 : cluster [DBG] pgmap v266: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:30:23.899 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:30:18 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:30:17.247173+0000 mgr.smithi040.ogcprl (mgr.14164) 281 : cluster [DBG] pgmap v267: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:30:23.899 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:30:20 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:30:19.247537+0000 mgr.smithi040.ogcprl (mgr.14164) 282 : cluster [DBG] pgmap v268: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:30:23.899 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:30:22 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:30:21.247924+0000 mgr.smithi040.ogcprl (mgr.14164) 283 : cluster [DBG] pgmap v269: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:30:23.899 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:26.291+0000 7ff04ef51700 1 -- 172.21.15.66:0/3260486409 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7ff03802ffb0 msgr2=0x7ff038032470 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:30:23.899 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:26.291+0000 7ff04ef51700 1 --2- 172.21.15.66:0/3260486409 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7ff03802ffb0 0x7ff038032470 secure :-1 s=READY pgs=109 cs=0 l=1 rev1=1 rx=0x7ff0340096a0 tx=0x7ff034006b40).stop 2023-11-17T05:30:23.899 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:26.291+0000 7ff04ef51700 1 -- 172.21.15.66:0/3260486409 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7ff0480f6510 msgr2=0x7ff04806b9a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:30:23.899 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:26.291+0000 7ff04ef51700 1 --2- 172.21.15.66:0/3260486409 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7ff0480f6510 0x7ff04806b9a0 secure :-1 s=READY pgs=288 cs=0 l=1 rev1=1 rx=0x7ff03c013510 tx=0x7ff03c00b040).stop 2023-11-17T05:30:23.899 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:26.291+0000 7ff04ef51700 1 -- 172.21.15.66:0/3260486409 shutdown_connections 2023-11-17T05:30:23.899 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:26.291+0000 7ff04ef51700 1 --2- 172.21.15.66:0/3260486409 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7ff03802ffb0 0x7ff038032470 unknown :-1 s=CLOSED pgs=109 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:30:23.900 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:26.291+0000 7ff04ef51700 1 --2- 172.21.15.66:0/3260486409 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7ff0480f6510 0x7ff04806b9a0 unknown :-1 s=CLOSED pgs=288 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:30:23.900 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:26.291+0000 7ff04ef51700 1 -- 172.21.15.66:0/3260486409 >> 172.21.15.66:0/3260486409 conn(0x7ff0480f1c70 msgr2=0x7ff0480f2760 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:30:23.900 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:26.291+0000 7ff04ef51700 1 -- 172.21.15.66:0/3260486409 shutdown_connections 2023-11-17T05:30:23.900 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:29:26.291+0000 7ff04ef51700 1 -- 172.21.15.66:0/3260486409 wait complete. 2023-11-17T05:30:23.900 INFO:teuthology.orchestra.run.smithi066.stderr:dumped monmap epoch 1 2023-11-17T05:30:24.739 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:30:24 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:30:23.248310+0000 mgr.smithi040.ogcprl (mgr.14164) 284 : cluster [DBG] pgmap v270: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:30:24.900 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2023-11-17T05:30:24.901 DEBUG:teuthology.orchestra.run.smithi066:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.4 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ed6470e2-8508-11ee-8dbe-212e2dc638e7 -- ceph mon dump -f json 2023-11-17T05:30:26.139 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:26.136+0000 7f88ea6a0700 1 -- 172.21.15.66:0/539977255 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f88e40f78e0 msgr2=0x7f88e40f7d00 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:30:26.139 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:26.136+0000 7f88ea6a0700 1 --2- 172.21.15.66:0/539977255 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f88e40f78e0 0x7f88e40f7d00 secure :-1 s=READY pgs=289 cs=0 l=1 rev1=1 rx=0x7f88e0009fb0 tx=0x7f88e0009b30).stop 2023-11-17T05:30:26.139 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:26.136+0000 7f88ea6a0700 1 -- 172.21.15.66:0/539977255 shutdown_connections 2023-11-17T05:30:26.139 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:26.136+0000 7f88ea6a0700 1 --2- 172.21.15.66:0/539977255 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f88e40f78e0 0x7f88e40f7d00 unknown :-1 s=CLOSED pgs=289 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:30:26.139 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:26.136+0000 7f88ea6a0700 1 -- 172.21.15.66:0/539977255 >> 172.21.15.66:0/539977255 conn(0x7f88e40f0120 msgr2=0x7f88e40f2540 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:30:26.139 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:26.136+0000 7f88ea6a0700 1 -- 172.21.15.66:0/539977255 shutdown_connections 2023-11-17T05:30:26.139 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:26.136+0000 7f88ea6a0700 1 -- 172.21.15.66:0/539977255 wait complete. 2023-11-17T05:30:26.139 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:26.137+0000 7f88ea6a0700 1 Processor -- start 2023-11-17T05:30:26.139 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:26.137+0000 7f88ea6a0700 1 -- start start 2023-11-17T05:30:26.140 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:26.137+0000 7f88ea6a0700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f88e40f78e0 0x7f88e41064e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:30:26.140 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:26.137+0000 7f88ea6a0700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f88e41069d0 con 0x7f88e40f78e0 2023-11-17T05:30:26.140 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:26.137+0000 7f88e969e700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f88e40f78e0 0x7f88e41064e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:30:26.140 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:26.137+0000 7f88e969e700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f88e40f78e0 0x7f88e41064e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.66:60684/0 (socket says 172.21.15.66:60684) 2023-11-17T05:30:26.140 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:26.137+0000 7f88e969e700 1 -- 172.21.15.66:0/2547369575 learned_addr learned my addr 172.21.15.66:0/2547369575 (peer_addr_for_me v2:172.21.15.66:0/0) 2023-11-17T05:30:26.140 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:26.137+0000 7f88e969e700 1 -- 172.21.15.66:0/2547369575 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f88e0005040 con 0x7f88e40f78e0 2023-11-17T05:30:26.140 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:26.138+0000 7f88e969e700 1 --2- 172.21.15.66:0/2547369575 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f88e40f78e0 0x7f88e41064e0 secure :-1 s=READY pgs=290 cs=0 l=1 rev1=1 rx=0x7f88e0013510 tx=0x7f88e000b040).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:30:26.140 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:26.138+0000 7f88d9ffb700 1 -- 172.21.15.66:0/2547369575 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f88e0009320 con 0x7f88e40f78e0 2023-11-17T05:30:26.140 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:26.138+0000 7f88d9ffb700 1 -- 172.21.15.66:0/2547369575 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(25 keys) v1 ==== 1074+0+0 (secure 0 0 0) 0x7f88e0013f00 con 0x7f88e40f78e0 2023-11-17T05:30:26.141 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:26.138+0000 7f88d9ffb700 1 -- 172.21.15.66:0/2547369575 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f88e0014dc0 con 0x7f88e40f78e0 2023-11-17T05:30:26.141 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:26.138+0000 7f88ea6a0700 1 -- 172.21.15.66:0/2547369575 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f88e4106bd0 con 0x7f88e40f78e0 2023-11-17T05:30:26.141 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:26.138+0000 7f88ea6a0700 1 -- 172.21.15.66:0/2547369575 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f88e4107060 con 0x7f88e40f78e0 2023-11-17T05:30:26.142 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:26.139+0000 7f88d9ffb700 1 -- 172.21.15.66:0/2547369575 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 17) v1 ==== 36269+0+0 (secure 0 0 0) 0x7f88e00095f0 con 0x7f88e40f78e0 2023-11-17T05:30:26.142 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:26.139+0000 7f88ea6a0700 1 -- 172.21.15.66:0/2547369575 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f88e4045ad0 con 0x7f88e40f78e0 2023-11-17T05:30:26.166 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:26.139+0000 7f88d9ffb700 1 --2- 172.21.15.66:0/2547369575 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f88d002ff60 0x7f88d0032420 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:30:26.166 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:26.139+0000 7f88d9ffb700 1 -- 172.21.15.66:0/2547369575 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(5..5 src has 1..5) v4 ==== 1270+0+0 (secure 0 0 0) 0x7f88e00410e0 con 0x7f88e40f78e0 2023-11-17T05:30:26.166 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:26.139+0000 7f88e8e9d700 1 --2- 172.21.15.66:0/2547369575 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f88d002ff60 0x7f88d0032420 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:30:26.166 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:26.140+0000 7f88e8e9d700 1 --2- 172.21.15.66:0/2547369575 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f88d002ff60 0x7f88d0032420 secure :-1 s=READY pgs=110 cs=0 l=1 rev1=1 rx=0x7f88d40096a0 tx=0x7f88d4006b40).ready entity=mgr.14164 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:30:26.167 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:26.144+0000 7f88d9ffb700 1 -- 172.21.15.66:0/2547369575 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+152212 (secure 0 0 0) 0x7f88e0014420 con 0x7f88e40f78e0 2023-11-17T05:30:26.365 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:26.365+0000 7f88ea6a0700 1 -- 172.21.15.66:0/2547369575 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "mon dump", "format": "json"} v 0) v1 -- 0x7f88e4107eb0 con 0x7f88e40f78e0 2023-11-17T05:30:26.365 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:26.365+0000 7f88d9ffb700 1 -- 172.21.15.66:0/2547369575 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mon_command_ack([{"prefix": "mon dump", "format": "json"}]=0 dumped monmap epoch 1 v1) v1 ==== 95+0+690 (secure 0 0 0) 0x7f88e001c920 con 0x7f88e40f78e0 2023-11-17T05:30:26.366 INFO:teuthology.orchestra.run.smithi066.stdout: 2023-11-17T05:30:26.366 INFO:teuthology.orchestra.run.smithi066.stdout:{"epoch":1,"fsid":"ed6470e2-8508-11ee-8dbe-212e2dc638e7","modified":"2023-11-17T05:20:13.934038Z","created":"2023-11-17T05:20:13.934038Z","min_mon_release":16,"min_mon_release_name":"pacific","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging"],"optional":[]},"mons":[{"rank":0,"name":"smithi040","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:3300","nonce":0},{"type":"v1","addr":"172.21.15.40:6789","nonce":0}]},"addr":"172.21.15.40:6789/0","public_addr":"172.21.15.40:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2023-11-17T05:30:26.367 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:26.367+0000 7f88ea6a0700 1 -- 172.21.15.66:0/2547369575 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f88d002ff60 msgr2=0x7f88d0032420 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:30:26.367 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:26.367+0000 7f88ea6a0700 1 --2- 172.21.15.66:0/2547369575 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f88d002ff60 0x7f88d0032420 secure :-1 s=READY pgs=110 cs=0 l=1 rev1=1 rx=0x7f88d40096a0 tx=0x7f88d4006b40).stop 2023-11-17T05:30:26.367 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:26.367+0000 7f88ea6a0700 1 -- 172.21.15.66:0/2547369575 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f88e40f78e0 msgr2=0x7f88e41064e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:30:26.368 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:26.367+0000 7f88ea6a0700 1 --2- 172.21.15.66:0/2547369575 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f88e40f78e0 0x7f88e41064e0 secure :-1 s=READY pgs=290 cs=0 l=1 rev1=1 rx=0x7f88e0013510 tx=0x7f88e000b040).stop 2023-11-17T05:30:26.368 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:26.367+0000 7f88ea6a0700 1 -- 172.21.15.66:0/2547369575 shutdown_connections 2023-11-17T05:30:26.368 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:26.367+0000 7f88ea6a0700 1 --2- 172.21.15.66:0/2547369575 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f88d002ff60 0x7f88d0032420 unknown :-1 s=CLOSED pgs=110 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:30:26.368 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:26.367+0000 7f88ea6a0700 1 --2- 172.21.15.66:0/2547369575 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f88e40f78e0 0x7f88e41064e0 unknown :-1 s=CLOSED pgs=290 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:30:26.368 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:26.367+0000 7f88ea6a0700 1 -- 172.21.15.66:0/2547369575 >> 172.21.15.66:0/2547369575 conn(0x7f88e40f0120 msgr2=0x7f88e40f0ba0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:30:26.368 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:26.367+0000 7f88ea6a0700 1 -- 172.21.15.66:0/2547369575 shutdown_connections 2023-11-17T05:30:26.368 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:26.367+0000 7f88ea6a0700 1 -- 172.21.15.66:0/2547369575 wait complete. 2023-11-17T05:30:26.368 INFO:teuthology.orchestra.run.smithi066.stderr:dumped monmap epoch 1 2023-11-17T05:30:26.739 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:30:26 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:30:25.248666+0000 mgr.smithi040.ogcprl (mgr.14164) 285 : cluster [DBG] pgmap v271: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:30:26.739 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:30:26 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:30:26.365959+0000 mon.smithi040 (mon.0) 245 : audit [DBG] from='client.? 172.21.15.66:0/2547369575' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-11-17T05:30:27.764 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2023-11-17T05:30:27.765 DEBUG:teuthology.orchestra.run.smithi066:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.4 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ed6470e2-8508-11ee-8dbe-212e2dc638e7 -- ceph mon dump -f json 2023-11-17T05:30:28.739 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:30:28 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:30:27.249032+0000 mgr.smithi040.ogcprl (mgr.14164) 286 : cluster [DBG] pgmap v272: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:30:28.964 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:28.963+0000 7efc8592f700 1 -- 172.21.15.66:0/755814015 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7efc800f5410 msgr2=0x7efc800f5830 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:30:28.964 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:28.963+0000 7efc8592f700 1 --2- 172.21.15.66:0/755814015 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7efc800f5410 0x7efc800f5830 secure :-1 s=READY pgs=291 cs=0 l=1 rev1=1 rx=0x7efc74009fb0 tx=0x7efc74009b30).stop 2023-11-17T05:30:28.964 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:28.963+0000 7efc8592f700 1 -- 172.21.15.66:0/755814015 shutdown_connections 2023-11-17T05:30:28.964 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:28.963+0000 7efc8592f700 1 --2- 172.21.15.66:0/755814015 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7efc800f5410 0x7efc800f5830 unknown :-1 s=CLOSED pgs=291 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:30:28.964 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:28.963+0000 7efc8592f700 1 -- 172.21.15.66:0/755814015 >> 172.21.15.66:0/755814015 conn(0x7efc800f0b70 msgr2=0x7efc800f2f90 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:30:28.964 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:28.963+0000 7efc8592f700 1 -- 172.21.15.66:0/755814015 shutdown_connections 2023-11-17T05:30:28.964 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:28.963+0000 7efc8592f700 1 -- 172.21.15.66:0/755814015 wait complete. 2023-11-17T05:30:28.966 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:28.964+0000 7efc8592f700 1 Processor -- start 2023-11-17T05:30:28.966 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:28.964+0000 7efc8592f700 1 -- start start 2023-11-17T05:30:28.966 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:28.964+0000 7efc8592f700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7efc800f5410 0x7efc80106720 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:30:28.976 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:28.964+0000 7efc8592f700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7efc80106c10 con 0x7efc800f5410 2023-11-17T05:30:28.977 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:28.965+0000 7efc7effd700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7efc800f5410 0x7efc80106720 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:30:28.977 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:28.965+0000 7efc7effd700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7efc800f5410 0x7efc80106720 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.66:60702/0 (socket says 172.21.15.66:60702) 2023-11-17T05:30:28.977 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:28.965+0000 7efc7effd700 1 -- 172.21.15.66:0/1600137677 learned_addr learned my addr 172.21.15.66:0/1600137677 (peer_addr_for_me v2:172.21.15.66:0/0) 2023-11-17T05:30:28.977 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:28.965+0000 7efc7effd700 1 -- 172.21.15.66:0/1600137677 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7efc74005040 con 0x7efc800f5410 2023-11-17T05:30:28.977 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:28.965+0000 7efc7effd700 1 --2- 172.21.15.66:0/1600137677 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7efc800f5410 0x7efc80106720 secure :-1 s=READY pgs=292 cs=0 l=1 rev1=1 rx=0x7efc74013510 tx=0x7efc7400b040).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:30:28.977 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:28.965+0000 7efc677fe700 1 -- 172.21.15.66:0/1600137677 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7efc74009320 con 0x7efc800f5410 2023-11-17T05:30:28.977 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:28.965+0000 7efc8592f700 1 -- 172.21.15.66:0/1600137677 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7efc80106e10 con 0x7efc800f5410 2023-11-17T05:30:28.977 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:28.965+0000 7efc677fe700 1 -- 172.21.15.66:0/1600137677 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(25 keys) v1 ==== 1074+0+0 (secure 0 0 0) 0x7efc74013f00 con 0x7efc800f5410 2023-11-17T05:30:28.977 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:28.965+0000 7efc677fe700 1 -- 172.21.15.66:0/1600137677 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7efc74014d90 con 0x7efc800f5410 2023-11-17T05:30:28.977 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:28.965+0000 7efc8592f700 1 -- 172.21.15.66:0/1600137677 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7efc801072a0 con 0x7efc800f5410 2023-11-17T05:30:28.983 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:28.966+0000 7efc677fe700 1 -- 172.21.15.66:0/1600137677 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 17) v1 ==== 36269+0+0 (secure 0 0 0) 0x7efc740095f0 con 0x7efc800f5410 2023-11-17T05:30:28.983 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:28.966+0000 7efc8592f700 1 -- 172.21.15.66:0/1600137677 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7efc80045ad0 con 0x7efc800f5410 2023-11-17T05:30:28.984 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:28.966+0000 7efc677fe700 1 --2- 172.21.15.66:0/1600137677 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7efc6802ff60 0x7efc68032420 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:30:28.984 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:28.966+0000 7efc677fe700 1 -- 172.21.15.66:0/1600137677 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(5..5 src has 1..5) v4 ==== 1270+0+0 (secure 0 0 0) 0x7efc74040ff0 con 0x7efc800f5410 2023-11-17T05:30:28.984 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:28.967+0000 7efc7e7fc700 1 --2- 172.21.15.66:0/1600137677 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7efc6802ff60 0x7efc68032420 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:30:28.985 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:28.967+0000 7efc7e7fc700 1 --2- 172.21.15.66:0/1600137677 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7efc6802ff60 0x7efc68032420 secure :-1 s=READY pgs=111 cs=0 l=1 rev1=1 rx=0x7efc700096a0 tx=0x7efc70006b40).ready entity=mgr.14164 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:30:28.985 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:28.971+0000 7efc677fe700 1 -- 172.21.15.66:0/1600137677 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+152212 (secure 0 0 0) 0x7efc74014420 con 0x7efc800f5410 2023-11-17T05:30:29.189 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:29.189+0000 7efc8592f700 1 -- 172.21.15.66:0/1600137677 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "mon dump", "format": "json"} v 0) v1 -- 0x7efc80108150 con 0x7efc800f5410 2023-11-17T05:30:29.189 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:29.189+0000 7efc677fe700 1 -- 172.21.15.66:0/1600137677 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mon_command_ack([{"prefix": "mon dump", "format": "json"}]=0 dumped monmap epoch 1 v1) v1 ==== 95+0+690 (secure 0 0 0) 0x7efc7401d7f0 con 0x7efc800f5410 2023-11-17T05:30:29.189 INFO:teuthology.orchestra.run.smithi066.stdout: 2023-11-17T05:30:29.190 INFO:teuthology.orchestra.run.smithi066.stdout:{"epoch":1,"fsid":"ed6470e2-8508-11ee-8dbe-212e2dc638e7","modified":"2023-11-17T05:20:13.934038Z","created":"2023-11-17T05:20:13.934038Z","min_mon_release":16,"min_mon_release_name":"pacific","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging"],"optional":[]},"mons":[{"rank":0,"name":"smithi040","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:3300","nonce":0},{"type":"v1","addr":"172.21.15.40:6789","nonce":0}]},"addr":"172.21.15.40:6789/0","public_addr":"172.21.15.40:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2023-11-17T05:30:29.191 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:29.191+0000 7efc8592f700 1 -- 172.21.15.66:0/1600137677 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7efc6802ff60 msgr2=0x7efc68032420 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:30:29.191 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:29.191+0000 7efc8592f700 1 --2- 172.21.15.66:0/1600137677 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7efc6802ff60 0x7efc68032420 secure :-1 s=READY pgs=111 cs=0 l=1 rev1=1 rx=0x7efc700096a0 tx=0x7efc70006b40).stop 2023-11-17T05:30:29.191 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:29.191+0000 7efc8592f700 1 -- 172.21.15.66:0/1600137677 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7efc800f5410 msgr2=0x7efc80106720 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:30:29.191 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:29.191+0000 7efc8592f700 1 --2- 172.21.15.66:0/1600137677 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7efc800f5410 0x7efc80106720 secure :-1 s=READY pgs=292 cs=0 l=1 rev1=1 rx=0x7efc74013510 tx=0x7efc7400b040).stop 2023-11-17T05:30:29.192 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:29.191+0000 7efc8592f700 1 -- 172.21.15.66:0/1600137677 shutdown_connections 2023-11-17T05:30:29.192 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:29.191+0000 7efc8592f700 1 --2- 172.21.15.66:0/1600137677 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7efc6802ff60 0x7efc68032420 unknown :-1 s=CLOSED pgs=111 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:30:29.192 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:29.191+0000 7efc8592f700 1 --2- 172.21.15.66:0/1600137677 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7efc800f5410 0x7efc80106720 unknown :-1 s=CLOSED pgs=292 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:30:29.192 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:29.191+0000 7efc8592f700 1 -- 172.21.15.66:0/1600137677 >> 172.21.15.66:0/1600137677 conn(0x7efc800f0b70 msgr2=0x7efc8005fe20 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:30:29.192 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:29.191+0000 7efc8592f700 1 -- 172.21.15.66:0/1600137677 shutdown_connections 2023-11-17T05:30:29.192 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:29.191+0000 7efc8592f700 1 -- 172.21.15.66:0/1600137677 wait complete. 2023-11-17T05:30:29.192 INFO:teuthology.orchestra.run.smithi066.stderr:dumped monmap epoch 1 2023-11-17T05:30:29.988 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:30:29 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:30:29.189946+0000 mon.smithi040 (mon.0) 246 : audit [DBG] from='client.? 172.21.15.66:0/1600137677' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-11-17T05:30:30.619 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2023-11-17T05:30:30.620 DEBUG:teuthology.orchestra.run.smithi066:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.4 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ed6470e2-8508-11ee-8dbe-212e2dc638e7 -- ceph mon dump -f json 2023-11-17T05:30:30.990 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:30:30 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:30:29.249433+0000 mgr.smithi040.ogcprl (mgr.14164) 287 : cluster [DBG] pgmap v273: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:30:31.728 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:31.725+0000 7ffbb68cf700 1 -- 172.21.15.66:0/3021230823 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7ffbb00f4740 msgr2=0x7ffbb00f4b60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:30:31.728 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:31.725+0000 7ffbb68cf700 1 --2- 172.21.15.66:0/3021230823 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7ffbb00f4740 0x7ffbb00f4b60 secure :-1 s=READY pgs=293 cs=0 l=1 rev1=1 rx=0x7ffba0009fb0 tx=0x7ffba0009b30).stop 2023-11-17T05:30:31.728 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:31.725+0000 7ffbb68cf700 1 -- 172.21.15.66:0/3021230823 shutdown_connections 2023-11-17T05:30:31.728 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:31.725+0000 7ffbb68cf700 1 --2- 172.21.15.66:0/3021230823 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7ffbb00f4740 0x7ffbb00f4b60 unknown :-1 s=CLOSED pgs=293 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:30:31.728 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:31.725+0000 7ffbb68cf700 1 -- 172.21.15.66:0/3021230823 >> 172.21.15.66:0/3021230823 conn(0x7ffbb00f0260 msgr2=0x7ffbb00f2680 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:30:31.728 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:31.725+0000 7ffbb68cf700 1 -- 172.21.15.66:0/3021230823 shutdown_connections 2023-11-17T05:30:31.728 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:31.726+0000 7ffbb68cf700 1 -- 172.21.15.66:0/3021230823 wait complete. 2023-11-17T05:30:31.728 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:31.726+0000 7ffbb68cf700 1 Processor -- start 2023-11-17T05:30:31.729 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:31.726+0000 7ffbb68cf700 1 -- start start 2023-11-17T05:30:31.729 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:31.726+0000 7ffbb68cf700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7ffbb00f4740 0x7ffbb01066d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:30:31.729 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:31.726+0000 7ffbb68cf700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7ffbb0106bc0 con 0x7ffbb00f4740 2023-11-17T05:30:31.729 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:31.727+0000 7ffbaffff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7ffbb00f4740 0x7ffbb01066d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:30:31.729 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:31.727+0000 7ffbaffff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7ffbb00f4740 0x7ffbb01066d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.66:60722/0 (socket says 172.21.15.66:60722) 2023-11-17T05:30:31.729 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:31.727+0000 7ffbaffff700 1 -- 172.21.15.66:0/2896515746 learned_addr learned my addr 172.21.15.66:0/2896515746 (peer_addr_for_me v2:172.21.15.66:0/0) 2023-11-17T05:30:31.729 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:31.727+0000 7ffbaffff700 1 -- 172.21.15.66:0/2896515746 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7ffba0005040 con 0x7ffbb00f4740 2023-11-17T05:30:31.729 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:31.727+0000 7ffbaffff700 1 --2- 172.21.15.66:0/2896515746 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7ffbb00f4740 0x7ffbb01066d0 secure :-1 s=READY pgs=294 cs=0 l=1 rev1=1 rx=0x7ffba0013510 tx=0x7ffba000b040).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:30:31.729 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:31.727+0000 7ffbacff9700 1 -- 172.21.15.66:0/2896515746 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7ffba0009320 con 0x7ffbb00f4740 2023-11-17T05:30:31.730 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:31.727+0000 7ffbb68cf700 1 -- 172.21.15.66:0/2896515746 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7ffbb0106dc0 con 0x7ffbb00f4740 2023-11-17T05:30:31.730 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:31.727+0000 7ffbacff9700 1 -- 172.21.15.66:0/2896515746 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(25 keys) v1 ==== 1074+0+0 (secure 0 0 0) 0x7ffba0013f00 con 0x7ffbb00f4740 2023-11-17T05:30:31.730 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:31.727+0000 7ffbacff9700 1 -- 172.21.15.66:0/2896515746 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7ffba0014d90 con 0x7ffbb00f4740 2023-11-17T05:30:31.730 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:31.727+0000 7ffbb68cf700 1 -- 172.21.15.66:0/2896515746 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7ffbb0107250 con 0x7ffbb00f4740 2023-11-17T05:30:31.731 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:31.728+0000 7ffbb68cf700 1 -- 172.21.15.66:0/2896515746 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7ffbb0045ad0 con 0x7ffbb00f4740 2023-11-17T05:30:31.731 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:31.728+0000 7ffbacff9700 1 -- 172.21.15.66:0/2896515746 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 17) v1 ==== 36269+0+0 (secure 0 0 0) 0x7ffba0014420 con 0x7ffbb00f4740 2023-11-17T05:30:31.731 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:31.728+0000 7ffbacff9700 1 --2- 172.21.15.66:0/2896515746 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7ffb9802ff60 0x7ffb98032420 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:30:31.731 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:31.728+0000 7ffbacff9700 1 -- 172.21.15.66:0/2896515746 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(5..5 src has 1..5) v4 ==== 1270+0+0 (secure 0 0 0) 0x7ffba0041050 con 0x7ffbb00f4740 2023-11-17T05:30:31.731 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:31.728+0000 7ffbaf7fe700 1 --2- 172.21.15.66:0/2896515746 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7ffb9802ff60 0x7ffb98032420 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:30:31.731 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:31.729+0000 7ffbaf7fe700 1 --2- 172.21.15.66:0/2896515746 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7ffb9802ff60 0x7ffb98032420 secure :-1 s=READY pgs=112 cs=0 l=1 rev1=1 rx=0x7ffba40096a0 tx=0x7ffba4006b40).ready entity=mgr.14164 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:30:31.733 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:31.733+0000 7ffbacff9700 1 -- 172.21.15.66:0/2896515746 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+152212 (secure 0 0 0) 0x7ffba00095f0 con 0x7ffbb00f4740 2023-11-17T05:30:31.951 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:31.950+0000 7ffbb68cf700 1 -- 172.21.15.66:0/2896515746 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "mon dump", "format": "json"} v 0) v1 -- 0x7ffbb01080a0 con 0x7ffbb00f4740 2023-11-17T05:30:31.952 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:31.951+0000 7ffbacff9700 1 -- 172.21.15.66:0/2896515746 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mon_command_ack([{"prefix": "mon dump", "format": "json"}]=0 dumped monmap epoch 1 v1) v1 ==== 95+0+690 (secure 0 0 0) 0x7ffba001db70 con 0x7ffbb00f4740 2023-11-17T05:30:31.952 INFO:teuthology.orchestra.run.smithi066.stdout: 2023-11-17T05:30:31.952 INFO:teuthology.orchestra.run.smithi066.stdout:{"epoch":1,"fsid":"ed6470e2-8508-11ee-8dbe-212e2dc638e7","modified":"2023-11-17T05:20:13.934038Z","created":"2023-11-17T05:20:13.934038Z","min_mon_release":16,"min_mon_release_name":"pacific","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging"],"optional":[]},"mons":[{"rank":0,"name":"smithi040","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:3300","nonce":0},{"type":"v1","addr":"172.21.15.40:6789","nonce":0}]},"addr":"172.21.15.40:6789/0","public_addr":"172.21.15.40:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2023-11-17T05:30:31.953 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:31.952+0000 7ffbb68cf700 1 -- 172.21.15.66:0/2896515746 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7ffb9802ff60 msgr2=0x7ffb98032420 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:30:31.953 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:31.952+0000 7ffbb68cf700 1 --2- 172.21.15.66:0/2896515746 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7ffb9802ff60 0x7ffb98032420 secure :-1 s=READY pgs=112 cs=0 l=1 rev1=1 rx=0x7ffba40096a0 tx=0x7ffba4006b40).stop 2023-11-17T05:30:31.954 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:31.952+0000 7ffbb68cf700 1 -- 172.21.15.66:0/2896515746 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7ffbb00f4740 msgr2=0x7ffbb01066d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:30:31.954 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:31.952+0000 7ffbb68cf700 1 --2- 172.21.15.66:0/2896515746 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7ffbb00f4740 0x7ffbb01066d0 secure :-1 s=READY pgs=294 cs=0 l=1 rev1=1 rx=0x7ffba0013510 tx=0x7ffba000b040).stop 2023-11-17T05:30:31.954 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:31.953+0000 7ffbb68cf700 1 -- 172.21.15.66:0/2896515746 shutdown_connections 2023-11-17T05:30:31.954 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:31.953+0000 7ffbb68cf700 1 --2- 172.21.15.66:0/2896515746 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7ffb9802ff60 0x7ffb98032420 unknown :-1 s=CLOSED pgs=112 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:30:31.954 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:31.953+0000 7ffbb68cf700 1 --2- 172.21.15.66:0/2896515746 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7ffbb00f4740 0x7ffbb01066d0 unknown :-1 s=CLOSED pgs=294 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:30:31.954 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:31.953+0000 7ffbb68cf700 1 -- 172.21.15.66:0/2896515746 >> 172.21.15.66:0/2896515746 conn(0x7ffbb00f0260 msgr2=0x7ffbb00f0d80 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:30:31.954 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:31.953+0000 7ffbb68cf700 1 -- 172.21.15.66:0/2896515746 shutdown_connections 2023-11-17T05:30:31.954 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:31.953+0000 7ffbb68cf700 1 -- 172.21.15.66:0/2896515746 wait complete. 2023-11-17T05:30:31.954 INFO:teuthology.orchestra.run.smithi066.stderr:dumped monmap epoch 1 2023-11-17T05:30:32.989 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:30:32 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:30:31.249852+0000 mgr.smithi040.ogcprl (mgr.14164) 288 : cluster [DBG] pgmap v274: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:30:32.989 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:30:32 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:30:31.951722+0000 mon.smithi040 (mon.0) 247 : audit [DBG] from='client.? 172.21.15.66:0/2896515746' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-11-17T05:30:33.334 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2023-11-17T05:30:33.334 DEBUG:teuthology.orchestra.run.smithi066:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.4 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ed6470e2-8508-11ee-8dbe-212e2dc638e7 -- ceph mon dump -f json 2023-11-17T05:30:34.505 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:34.503+0000 7fa2f8630700 1 -- 172.21.15.66:0/1546273632 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fa2f00f5410 msgr2=0x7fa2f00f5830 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:30:34.506 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:34.503+0000 7fa2f8630700 1 --2- 172.21.15.66:0/1546273632 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fa2f00f5410 0x7fa2f00f5830 secure :-1 s=READY pgs=295 cs=0 l=1 rev1=1 rx=0x7fa2ec009fb0 tx=0x7fa2ec009b30).stop 2023-11-17T05:30:34.506 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:34.503+0000 7fa2f8630700 1 -- 172.21.15.66:0/1546273632 shutdown_connections 2023-11-17T05:30:34.506 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:34.503+0000 7fa2f8630700 1 --2- 172.21.15.66:0/1546273632 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fa2f00f5410 0x7fa2f00f5830 unknown :-1 s=CLOSED pgs=295 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:30:34.506 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:34.503+0000 7fa2f8630700 1 -- 172.21.15.66:0/1546273632 >> 172.21.15.66:0/1546273632 conn(0x7fa2f00f0b70 msgr2=0x7fa2f00f2f90 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:30:34.506 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:34.503+0000 7fa2f8630700 1 -- 172.21.15.66:0/1546273632 shutdown_connections 2023-11-17T05:30:34.506 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:34.503+0000 7fa2f8630700 1 -- 172.21.15.66:0/1546273632 wait complete. 2023-11-17T05:30:34.506 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:34.504+0000 7fa2f8630700 1 Processor -- start 2023-11-17T05:30:34.506 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:34.504+0000 7fa2f8630700 1 -- start start 2023-11-17T05:30:34.506 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:34.504+0000 7fa2f8630700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fa2f00f5410 0x7fa2f00fd3a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:30:34.506 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:34.504+0000 7fa2f8630700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fa2f00fb820 con 0x7fa2f00f5410 2023-11-17T05:30:34.507 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:34.504+0000 7fa2f63cc700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fa2f00f5410 0x7fa2f00fd3a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:30:34.507 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:34.504+0000 7fa2f63cc700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fa2f00f5410 0x7fa2f00fd3a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.66:51772/0 (socket says 172.21.15.66:51772) 2023-11-17T05:30:34.507 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:34.504+0000 7fa2f63cc700 1 -- 172.21.15.66:0/281358223 learned_addr learned my addr 172.21.15.66:0/281358223 (peer_addr_for_me v2:172.21.15.66:0/0) 2023-11-17T05:30:34.507 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:34.505+0000 7fa2f63cc700 1 -- 172.21.15.66:0/281358223 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fa2ec005040 con 0x7fa2f00f5410 2023-11-17T05:30:34.507 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:34.505+0000 7fa2f63cc700 1 --2- 172.21.15.66:0/281358223 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fa2f00f5410 0x7fa2f00fd3a0 secure :-1 s=READY pgs=296 cs=0 l=1 rev1=1 rx=0x7fa2ec013510 tx=0x7fa2ec00b040).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:30:34.507 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:34.505+0000 7fa2e2ffd700 1 -- 172.21.15.66:0/281358223 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7fa2ec009320 con 0x7fa2f00f5410 2023-11-17T05:30:34.507 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:34.505+0000 7fa2f8630700 1 -- 172.21.15.66:0/281358223 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fa2f00fb9c0 con 0x7fa2f00f5410 2023-11-17T05:30:34.507 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:34.505+0000 7fa2e2ffd700 1 -- 172.21.15.66:0/281358223 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(25 keys) v1 ==== 1074+0+0 (secure 0 0 0) 0x7fa2ec013f00 con 0x7fa2f00f5410 2023-11-17T05:30:34.507 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:34.505+0000 7fa2f8630700 1 -- 172.21.15.66:0/281358223 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fa2f00fbe50 con 0x7fa2f00f5410 2023-11-17T05:30:34.507 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:34.505+0000 7fa2e2ffd700 1 -- 172.21.15.66:0/281358223 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7fa2ec014d90 con 0x7fa2f00f5410 2023-11-17T05:30:34.508 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:34.506+0000 7fa2e2ffd700 1 -- 172.21.15.66:0/281358223 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 17) v1 ==== 36269+0+0 (secure 0 0 0) 0x7fa2ec014420 con 0x7fa2f00f5410 2023-11-17T05:30:34.508 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:34.506+0000 7fa2f8630700 1 -- 172.21.15.66:0/281358223 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fa2f0059100 con 0x7fa2f00f5410 2023-11-17T05:30:34.508 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:34.506+0000 7fa2e2ffd700 1 --2- 172.21.15.66:0/281358223 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fa2dc02ffb0 0x7fa2dc032470 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:30:34.508 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:34.506+0000 7fa2e2ffd700 1 -- 172.21.15.66:0/281358223 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(5..5 src has 1..5) v4 ==== 1270+0+0 (secure 0 0 0) 0x7fa2ec040fd0 con 0x7fa2f00f5410 2023-11-17T05:30:34.508 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:34.506+0000 7fa2f5bcb700 1 --2- 172.21.15.66:0/281358223 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fa2dc02ffb0 0x7fa2dc032470 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:30:34.509 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:34.506+0000 7fa2f5bcb700 1 --2- 172.21.15.66:0/281358223 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fa2dc02ffb0 0x7fa2dc032470 secure :-1 s=READY pgs=113 cs=0 l=1 rev1=1 rx=0x7fa2e40096a0 tx=0x7fa2e4006b40).ready entity=mgr.14164 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:30:34.511 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:34.511+0000 7fa2e2ffd700 1 -- 172.21.15.66:0/281358223 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+152212 (secure 0 0 0) 0x7fa2ec06d2b0 con 0x7fa2f00f5410 2023-11-17T05:30:34.731 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:34.731+0000 7fa2f8630700 1 -- 172.21.15.66:0/281358223 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "mon dump", "format": "json"} v 0) v1 -- 0x7fa2f00fae60 con 0x7fa2f00f5410 2023-11-17T05:30:34.732 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:34.732+0000 7fa2e2ffd700 1 -- 172.21.15.66:0/281358223 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mon_command_ack([{"prefix": "mon dump", "format": "json"}]=0 dumped monmap epoch 1 v1) v1 ==== 95+0+690 (secure 0 0 0) 0x7fa2ec0095f0 con 0x7fa2f00f5410 2023-11-17T05:30:34.732 INFO:teuthology.orchestra.run.smithi066.stdout: 2023-11-17T05:30:34.733 INFO:teuthology.orchestra.run.smithi066.stdout:{"epoch":1,"fsid":"ed6470e2-8508-11ee-8dbe-212e2dc638e7","modified":"2023-11-17T05:20:13.934038Z","created":"2023-11-17T05:20:13.934038Z","min_mon_release":16,"min_mon_release_name":"pacific","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging"],"optional":[]},"mons":[{"rank":0,"name":"smithi040","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:3300","nonce":0},{"type":"v1","addr":"172.21.15.40:6789","nonce":0}]},"addr":"172.21.15.40:6789/0","public_addr":"172.21.15.40:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2023-11-17T05:30:34.734 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:34.734+0000 7fa2f8630700 1 -- 172.21.15.66:0/281358223 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fa2dc02ffb0 msgr2=0x7fa2dc032470 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:30:34.734 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:34.734+0000 7fa2f8630700 1 --2- 172.21.15.66:0/281358223 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fa2dc02ffb0 0x7fa2dc032470 secure :-1 s=READY pgs=113 cs=0 l=1 rev1=1 rx=0x7fa2e40096a0 tx=0x7fa2e4006b40).stop 2023-11-17T05:30:34.734 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:34.734+0000 7fa2f8630700 1 -- 172.21.15.66:0/281358223 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fa2f00f5410 msgr2=0x7fa2f00fd3a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:30:34.735 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:34.734+0000 7fa2f8630700 1 --2- 172.21.15.66:0/281358223 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fa2f00f5410 0x7fa2f00fd3a0 secure :-1 s=READY pgs=296 cs=0 l=1 rev1=1 rx=0x7fa2ec013510 tx=0x7fa2ec00b040).stop 2023-11-17T05:30:34.735 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:34.734+0000 7fa2f8630700 1 -- 172.21.15.66:0/281358223 shutdown_connections 2023-11-17T05:30:34.735 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:34.734+0000 7fa2f8630700 1 --2- 172.21.15.66:0/281358223 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fa2dc02ffb0 0x7fa2dc032470 unknown :-1 s=CLOSED pgs=113 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:30:34.735 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:34.734+0000 7fa2f8630700 1 --2- 172.21.15.66:0/281358223 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fa2f00f5410 0x7fa2f00fd3a0 unknown :-1 s=CLOSED pgs=296 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:30:34.735 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:34.734+0000 7fa2f8630700 1 -- 172.21.15.66:0/281358223 >> 172.21.15.66:0/281358223 conn(0x7fa2f00f0b70 msgr2=0x7fa2f005fe00 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:30:34.735 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:34.734+0000 7fa2f8630700 1 -- 172.21.15.66:0/281358223 shutdown_connections 2023-11-17T05:30:34.735 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:34.734+0000 7fa2f8630700 1 -- 172.21.15.66:0/281358223 wait complete. 2023-11-17T05:30:34.735 INFO:teuthology.orchestra.run.smithi066.stderr:dumped monmap epoch 1 2023-11-17T05:30:34.989 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:30:34 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:30:33.250088+0000 mgr.smithi040.ogcprl (mgr.14164) 289 : cluster [DBG] pgmap v275: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:30:35.989 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:30:35 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:30:34.732347+0000 mon.smithi040 (mon.0) 248 : audit [DBG] from='client.? 172.21.15.66:0/281358223' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-11-17T05:30:36.121 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2023-11-17T05:30:36.122 DEBUG:teuthology.orchestra.run.smithi066:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.4 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ed6470e2-8508-11ee-8dbe-212e2dc638e7 -- ceph mon dump -f json 2023-11-17T05:30:36.988 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:30:36 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:30:35.250373+0000 mgr.smithi040.ogcprl (mgr.14164) 290 : cluster [DBG] pgmap v276: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:30:37.337 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:37.336+0000 7fd63b151700 1 -- 172.21.15.66:0/3655312755 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fd6340f47c0 msgr2=0x7fd6340f4be0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:30:37.337 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:37.336+0000 7fd63b151700 1 --2- 172.21.15.66:0/3655312755 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fd6340f47c0 0x7fd6340f4be0 secure :-1 s=READY pgs=297 cs=0 l=1 rev1=1 rx=0x7fd628009fb0 tx=0x7fd628009b30).stop 2023-11-17T05:30:37.337 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:37.336+0000 7fd63b151700 1 -- 172.21.15.66:0/3655312755 shutdown_connections 2023-11-17T05:30:37.337 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:37.336+0000 7fd63b151700 1 --2- 172.21.15.66:0/3655312755 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fd6340f47c0 0x7fd6340f4be0 unknown :-1 s=CLOSED pgs=297 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:30:37.337 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:37.336+0000 7fd63b151700 1 -- 172.21.15.66:0/3655312755 >> 172.21.15.66:0/3655312755 conn(0x7fd6340f02e0 msgr2=0x7fd6340f2700 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:30:37.337 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:37.336+0000 7fd63b151700 1 -- 172.21.15.66:0/3655312755 shutdown_connections 2023-11-17T05:30:37.337 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:37.336+0000 7fd63b151700 1 -- 172.21.15.66:0/3655312755 wait complete. 2023-11-17T05:30:37.337 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:37.336+0000 7fd63b151700 1 Processor -- start 2023-11-17T05:30:37.337 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:37.336+0000 7fd63b151700 1 -- start start 2023-11-17T05:30:37.338 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:37.337+0000 7fd63b151700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fd6340f47c0 0x7fd6341066f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:30:37.338 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:37.337+0000 7fd63b151700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fd634106be0 con 0x7fd6340f47c0 2023-11-17T05:30:37.338 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:37.337+0000 7fd638eed700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fd6340f47c0 0x7fd6341066f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:30:37.338 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:37.337+0000 7fd638eed700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fd6340f47c0 0x7fd6341066f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.66:51784/0 (socket says 172.21.15.66:51784) 2023-11-17T05:30:37.339 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:37.337+0000 7fd638eed700 1 -- 172.21.15.66:0/1880560877 learned_addr learned my addr 172.21.15.66:0/1880560877 (peer_addr_for_me v2:172.21.15.66:0/0) 2023-11-17T05:30:37.339 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:37.337+0000 7fd638eed700 1 -- 172.21.15.66:0/1880560877 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fd628005040 con 0x7fd6340f47c0 2023-11-17T05:30:37.339 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:37.337+0000 7fd638eed700 1 --2- 172.21.15.66:0/1880560877 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fd6340f47c0 0x7fd6341066f0 secure :-1 s=READY pgs=298 cs=0 l=1 rev1=1 rx=0x7fd628013510 tx=0x7fd62800b040).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:30:37.339 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:37.338+0000 7fd6317fa700 1 -- 172.21.15.66:0/1880560877 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7fd628009320 con 0x7fd6340f47c0 2023-11-17T05:30:37.339 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:37.338+0000 7fd63b151700 1 -- 172.21.15.66:0/1880560877 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fd634106de0 con 0x7fd6340f47c0 2023-11-17T05:30:37.339 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:37.338+0000 7fd6317fa700 1 -- 172.21.15.66:0/1880560877 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(25 keys) v1 ==== 1074+0+0 (secure 0 0 0) 0x7fd628013f00 con 0x7fd6340f47c0 2023-11-17T05:30:37.339 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:37.338+0000 7fd6317fa700 1 -- 172.21.15.66:0/1880560877 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7fd628014d90 con 0x7fd6340f47c0 2023-11-17T05:30:37.339 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:37.338+0000 7fd63b151700 1 -- 172.21.15.66:0/1880560877 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fd634107270 con 0x7fd6340f47c0 2023-11-17T05:30:37.340 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:37.339+0000 7fd63b151700 1 -- 172.21.15.66:0/1880560877 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fd634045ad0 con 0x7fd6340f47c0 2023-11-17T05:30:37.340 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:37.339+0000 7fd6317fa700 1 -- 172.21.15.66:0/1880560877 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 17) v1 ==== 36269+0+0 (secure 0 0 0) 0x7fd6280095f0 con 0x7fd6340f47c0 2023-11-17T05:30:37.340 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:37.339+0000 7fd6317fa700 1 --2- 172.21.15.66:0/1880560877 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fd62402ffb0 0x7fd624032470 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:30:37.340 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:37.339+0000 7fd6317fa700 1 -- 172.21.15.66:0/1880560877 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(5..5 src has 1..5) v4 ==== 1270+0+0 (secure 0 0 0) 0x7fd628040f90 con 0x7fd6340f47c0 2023-11-17T05:30:37.340 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:37.339+0000 7fd633fff700 1 --2- 172.21.15.66:0/1880560877 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fd62402ffb0 0x7fd624032470 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:30:37.340 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:37.340+0000 7fd633fff700 1 --2- 172.21.15.66:0/1880560877 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fd62402ffb0 0x7fd624032470 secure :-1 s=READY pgs=114 cs=0 l=1 rev1=1 rx=0x7fd6200096a0 tx=0x7fd620006b40).ready entity=mgr.14164 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:30:37.344 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:37.344+0000 7fd6317fa700 1 -- 172.21.15.66:0/1880560877 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+152212 (secure 0 0 0) 0x7fd628014420 con 0x7fd6340f47c0 2023-11-17T05:30:37.564 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:37.563+0000 7fd63b151700 1 -- 172.21.15.66:0/1880560877 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "mon dump", "format": "json"} v 0) v1 -- 0x7fd634108120 con 0x7fd6340f47c0 2023-11-17T05:30:37.565 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:37.564+0000 7fd6317fa700 1 -- 172.21.15.66:0/1880560877 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mon_command_ack([{"prefix": "mon dump", "format": "json"}]=0 dumped monmap epoch 1 v1) v1 ==== 95+0+690 (secure 0 0 0) 0x7fd628010230 con 0x7fd6340f47c0 2023-11-17T05:30:37.566 INFO:teuthology.orchestra.run.smithi066.stdout: 2023-11-17T05:30:37.566 INFO:teuthology.orchestra.run.smithi066.stdout:{"epoch":1,"fsid":"ed6470e2-8508-11ee-8dbe-212e2dc638e7","modified":"2023-11-17T05:20:13.934038Z","created":"2023-11-17T05:20:13.934038Z","min_mon_release":16,"min_mon_release_name":"pacific","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging"],"optional":[]},"mons":[{"rank":0,"name":"smithi040","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:3300","nonce":0},{"type":"v1","addr":"172.21.15.40:6789","nonce":0}]},"addr":"172.21.15.40:6789/0","public_addr":"172.21.15.40:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2023-11-17T05:30:37.568 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:37.566+0000 7fd63b151700 1 -- 172.21.15.66:0/1880560877 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fd62402ffb0 msgr2=0x7fd624032470 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:30:37.568 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:37.566+0000 7fd63b151700 1 --2- 172.21.15.66:0/1880560877 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fd62402ffb0 0x7fd624032470 secure :-1 s=READY pgs=114 cs=0 l=1 rev1=1 rx=0x7fd6200096a0 tx=0x7fd620006b40).stop 2023-11-17T05:30:37.568 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:37.566+0000 7fd63b151700 1 -- 172.21.15.66:0/1880560877 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fd6340f47c0 msgr2=0x7fd6341066f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:30:37.568 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:37.566+0000 7fd63b151700 1 --2- 172.21.15.66:0/1880560877 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fd6340f47c0 0x7fd6341066f0 secure :-1 s=READY pgs=298 cs=0 l=1 rev1=1 rx=0x7fd628013510 tx=0x7fd62800b040).stop 2023-11-17T05:30:37.568 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:37.566+0000 7fd63b151700 1 -- 172.21.15.66:0/1880560877 shutdown_connections 2023-11-17T05:30:37.568 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:37.566+0000 7fd63b151700 1 --2- 172.21.15.66:0/1880560877 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fd62402ffb0 0x7fd624032470 unknown :-1 s=CLOSED pgs=114 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:30:37.569 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:37.566+0000 7fd63b151700 1 --2- 172.21.15.66:0/1880560877 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fd6340f47c0 0x7fd6341066f0 unknown :-1 s=CLOSED pgs=298 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:30:37.569 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:37.566+0000 7fd63b151700 1 -- 172.21.15.66:0/1880560877 >> 172.21.15.66:0/1880560877 conn(0x7fd6340f02e0 msgr2=0x7fd6340f0dd0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:30:37.569 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:37.566+0000 7fd63b151700 1 -- 172.21.15.66:0/1880560877 shutdown_connections 2023-11-17T05:30:37.569 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:37.566+0000 7fd63b151700 1 -- 172.21.15.66:0/1880560877 wait complete. 2023-11-17T05:30:37.569 INFO:teuthology.orchestra.run.smithi066.stderr:dumped monmap epoch 1 2023-11-17T05:30:38.988 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:30:38 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:30:37.250677+0000 mgr.smithi040.ogcprl (mgr.14164) 291 : cluster [DBG] pgmap v277: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:30:38.989 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:30:38 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:30:37.564978+0000 mon.smithi040 (mon.0) 249 : audit [DBG] from='client.? 172.21.15.66:0/1880560877' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-11-17T05:30:38.995 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2023-11-17T05:30:38.995 DEBUG:teuthology.orchestra.run.smithi066:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.4 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ed6470e2-8508-11ee-8dbe-212e2dc638e7 -- ceph mon dump -f json 2023-11-17T05:30:40.162 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:40.161+0000 7f119f83c700 1 -- 172.21.15.66:0/2140322809 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f11980f5410 msgr2=0x7f11980f5830 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:30:40.162 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:40.161+0000 7f119f83c700 1 --2- 172.21.15.66:0/2140322809 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f11980f5410 0x7f11980f5830 secure :-1 s=READY pgs=299 cs=0 l=1 rev1=1 rx=0x7f1194009fb0 tx=0x7f1194009b30).stop 2023-11-17T05:30:40.162 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:40.161+0000 7f119f83c700 1 -- 172.21.15.66:0/2140322809 shutdown_connections 2023-11-17T05:30:40.162 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:40.161+0000 7f119f83c700 1 --2- 172.21.15.66:0/2140322809 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f11980f5410 0x7f11980f5830 unknown :-1 s=CLOSED pgs=299 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:30:40.162 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:40.161+0000 7f119f83c700 1 -- 172.21.15.66:0/2140322809 >> 172.21.15.66:0/2140322809 conn(0x7f11980f0b70 msgr2=0x7f11980f2f90 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:30:40.163 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:40.161+0000 7f119f83c700 1 -- 172.21.15.66:0/2140322809 shutdown_connections 2023-11-17T05:30:40.163 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:40.161+0000 7f119f83c700 1 -- 172.21.15.66:0/2140322809 wait complete. 2023-11-17T05:30:40.164 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:40.162+0000 7f119f83c700 1 Processor -- start 2023-11-17T05:30:40.164 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:40.162+0000 7f119f83c700 1 -- start start 2023-11-17T05:30:40.164 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:40.162+0000 7f119f83c700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f11980f5410 0x7f1198105630 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:30:40.165 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:40.162+0000 7f119f83c700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f1198105b20 con 0x7f11980f5410 2023-11-17T05:30:40.165 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:40.163+0000 7f119d5d8700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f11980f5410 0x7f1198105630 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:30:40.165 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:40.163+0000 7f119d5d8700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f11980f5410 0x7f1198105630 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.66:51796/0 (socket says 172.21.15.66:51796) 2023-11-17T05:30:40.165 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:40.163+0000 7f119d5d8700 1 -- 172.21.15.66:0/348303878 learned_addr learned my addr 172.21.15.66:0/348303878 (peer_addr_for_me v2:172.21.15.66:0/0) 2023-11-17T05:30:40.165 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:40.163+0000 7f119d5d8700 1 -- 172.21.15.66:0/348303878 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f1194005040 con 0x7f11980f5410 2023-11-17T05:30:40.165 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:40.163+0000 7f119d5d8700 1 --2- 172.21.15.66:0/348303878 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f11980f5410 0x7f1198105630 secure :-1 s=READY pgs=300 cs=0 l=1 rev1=1 rx=0x7f1194013510 tx=0x7f119400b040).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:30:40.166 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:40.163+0000 7f1189ffb700 1 -- 172.21.15.66:0/348303878 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f1194009320 con 0x7f11980f5410 2023-11-17T05:30:40.166 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:40.163+0000 7f119f83c700 1 -- 172.21.15.66:0/348303878 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f1198105d20 con 0x7f11980f5410 2023-11-17T05:30:40.166 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:40.164+0000 7f119f83c700 1 -- 172.21.15.66:0/348303878 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f11981061b0 con 0x7f11980f5410 2023-11-17T05:30:40.166 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:40.164+0000 7f1189ffb700 1 -- 172.21.15.66:0/348303878 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(25 keys) v1 ==== 1074+0+0 (secure 0 0 0) 0x7f1194013f00 con 0x7f11980f5410 2023-11-17T05:30:40.166 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:40.164+0000 7f1189ffb700 1 -- 172.21.15.66:0/348303878 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f1194014d60 con 0x7f11980f5410 2023-11-17T05:30:40.168 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:40.165+0000 7f1189ffb700 1 -- 172.21.15.66:0/348303878 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 17) v1 ==== 36269+0+0 (secure 0 0 0) 0x7f11940263a0 con 0x7f11980f5410 2023-11-17T05:30:40.168 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:40.165+0000 7f1189ffb700 1 --2- 172.21.15.66:0/348303878 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f118402ff60 0x7f1184032420 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:30:40.168 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:40.165+0000 7f1189ffb700 1 -- 172.21.15.66:0/348303878 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(5..5 src has 1..5) v4 ==== 1270+0+0 (secure 0 0 0) 0x7f11940413d0 con 0x7f11980f5410 2023-11-17T05:30:40.169 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:40.165+0000 7f119cdd7700 1 --2- 172.21.15.66:0/348303878 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f118402ff60 0x7f1184032420 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:30:40.169 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:40.165+0000 7f119f83c700 1 -- 172.21.15.66:0/348303878 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f1198059100 con 0x7f11980f5410 2023-11-17T05:30:40.169 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:40.165+0000 7f119cdd7700 1 --2- 172.21.15.66:0/348303878 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f118402ff60 0x7f1184032420 secure :-1 s=READY pgs=115 cs=0 l=1 rev1=1 rx=0x7f118c0096a0 tx=0x7f118c006b40).ready entity=mgr.14164 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:30:40.170 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:40.170+0000 7f1189ffb700 1 -- 172.21.15.66:0/348303878 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+152212 (secure 0 0 0) 0x7f1194012510 con 0x7f11980f5410 2023-11-17T05:30:40.391 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:40.391+0000 7f119f83c700 1 -- 172.21.15.66:0/348303878 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "mon dump", "format": "json"} v 0) v1 -- 0x7f11980fae60 con 0x7f11980f5410 2023-11-17T05:30:40.392 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:40.392+0000 7f1189ffb700 1 -- 172.21.15.66:0/348303878 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mon_command_ack([{"prefix": "mon dump", "format": "json"}]=0 dumped monmap epoch 1 v1) v1 ==== 95+0+690 (secure 0 0 0) 0x7f119402b030 con 0x7f11980f5410 2023-11-17T05:30:40.392 INFO:teuthology.orchestra.run.smithi066.stdout: 2023-11-17T05:30:40.392 INFO:teuthology.orchestra.run.smithi066.stdout:{"epoch":1,"fsid":"ed6470e2-8508-11ee-8dbe-212e2dc638e7","modified":"2023-11-17T05:20:13.934038Z","created":"2023-11-17T05:20:13.934038Z","min_mon_release":16,"min_mon_release_name":"pacific","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging"],"optional":[]},"mons":[{"rank":0,"name":"smithi040","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:3300","nonce":0},{"type":"v1","addr":"172.21.15.40:6789","nonce":0}]},"addr":"172.21.15.40:6789/0","public_addr":"172.21.15.40:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2023-11-17T05:30:40.394 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:40.393+0000 7f119f83c700 1 -- 172.21.15.66:0/348303878 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f118402ff60 msgr2=0x7f1184032420 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:30:40.394 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:40.393+0000 7f119f83c700 1 --2- 172.21.15.66:0/348303878 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f118402ff60 0x7f1184032420 secure :-1 s=READY pgs=115 cs=0 l=1 rev1=1 rx=0x7f118c0096a0 tx=0x7f118c006b40).stop 2023-11-17T05:30:40.394 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:40.393+0000 7f119f83c700 1 -- 172.21.15.66:0/348303878 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f11980f5410 msgr2=0x7f1198105630 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:30:40.394 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:40.393+0000 7f119f83c700 1 --2- 172.21.15.66:0/348303878 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f11980f5410 0x7f1198105630 secure :-1 s=READY pgs=300 cs=0 l=1 rev1=1 rx=0x7f1194013510 tx=0x7f119400b040).stop 2023-11-17T05:30:40.394 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:40.393+0000 7f119f83c700 1 -- 172.21.15.66:0/348303878 shutdown_connections 2023-11-17T05:30:40.394 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:40.393+0000 7f119f83c700 1 --2- 172.21.15.66:0/348303878 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f118402ff60 0x7f1184032420 unknown :-1 s=CLOSED pgs=115 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:30:40.394 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:40.393+0000 7f119f83c700 1 --2- 172.21.15.66:0/348303878 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f11980f5410 0x7f1198105630 unknown :-1 s=CLOSED pgs=300 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:30:40.394 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:40.393+0000 7f119f83c700 1 -- 172.21.15.66:0/348303878 >> 172.21.15.66:0/348303878 conn(0x7f11980f0b70 msgr2=0x7f11980f1660 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:30:40.394 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:40.393+0000 7f119f83c700 1 -- 172.21.15.66:0/348303878 shutdown_connections 2023-11-17T05:30:40.394 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:40.393+0000 7f119f83c700 1 -- 172.21.15.66:0/348303878 wait complete. 2023-11-17T05:30:40.395 INFO:teuthology.orchestra.run.smithi066.stderr:dumped monmap epoch 1 2023-11-17T05:30:40.988 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:30:40 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:30:39.251021+0000 mgr.smithi040.ogcprl (mgr.14164) 292 : cluster [DBG] pgmap v278: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:30:40.989 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:30:40 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:30:40.392181+0000 mon.smithi040 (mon.0) 250 : audit [DBG] from='client.? 172.21.15.66:0/348303878' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-11-17T05:30:41.780 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2023-11-17T05:30:41.780 DEBUG:teuthology.orchestra.run.smithi066:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.4 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ed6470e2-8508-11ee-8dbe-212e2dc638e7 -- ceph mon dump -f json 2023-11-17T05:30:42.974 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:42.973+0000 7f9b5af38700 1 -- 172.21.15.66:0/1400643181 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f9b540f51a0 msgr2=0x7f9b540f55c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:30:42.975 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:42.973+0000 7f9b5af38700 1 --2- 172.21.15.66:0/1400643181 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f9b540f51a0 0x7f9b540f55c0 secure :-1 s=READY pgs=301 cs=0 l=1 rev1=1 rx=0x7f9b50009fb0 tx=0x7f9b50009b30).stop 2023-11-17T05:30:42.975 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:42.973+0000 7f9b5af38700 1 -- 172.21.15.66:0/1400643181 shutdown_connections 2023-11-17T05:30:42.975 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:42.973+0000 7f9b5af38700 1 --2- 172.21.15.66:0/1400643181 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f9b540f51a0 0x7f9b540f55c0 unknown :-1 s=CLOSED pgs=301 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:30:42.975 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:42.973+0000 7f9b5af38700 1 -- 172.21.15.66:0/1400643181 >> 172.21.15.66:0/1400643181 conn(0x7f9b540f09a0 msgr2=0x7f9b540f2dc0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:30:42.975 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:42.974+0000 7f9b5af38700 1 -- 172.21.15.66:0/1400643181 shutdown_connections 2023-11-17T05:30:42.975 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:42.974+0000 7f9b5af38700 1 -- 172.21.15.66:0/1400643181 wait complete. 2023-11-17T05:30:42.975 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:42.974+0000 7f9b5af38700 1 Processor -- start 2023-11-17T05:30:42.975 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:42.974+0000 7f9b5af38700 1 -- start start 2023-11-17T05:30:42.975 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:42.974+0000 7f9b5af38700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f9b540f51a0 0x7f9b541064b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:30:42.975 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:42.974+0000 7f9b5af38700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f9b541069a0 con 0x7f9b540f51a0 2023-11-17T05:30:42.977 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:42.975+0000 7f9b59f36700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f9b540f51a0 0x7f9b541064b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:30:42.977 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:42.975+0000 7f9b59f36700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f9b540f51a0 0x7f9b541064b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.66:39908/0 (socket says 172.21.15.66:39908) 2023-11-17T05:30:42.977 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:42.975+0000 7f9b59f36700 1 -- 172.21.15.66:0/1307378755 learned_addr learned my addr 172.21.15.66:0/1307378755 (peer_addr_for_me v2:172.21.15.66:0/0) 2023-11-17T05:30:42.977 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:42.975+0000 7f9b59f36700 1 -- 172.21.15.66:0/1307378755 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f9b50005040 con 0x7f9b540f51a0 2023-11-17T05:30:42.977 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:42.975+0000 7f9b59f36700 1 --2- 172.21.15.66:0/1307378755 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f9b540f51a0 0x7f9b541064b0 secure :-1 s=READY pgs=302 cs=0 l=1 rev1=1 rx=0x7f9b50013510 tx=0x7f9b5000b040).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:30:42.977 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:42.975+0000 7f9b467fc700 1 -- 172.21.15.66:0/1307378755 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f9b50009320 con 0x7f9b540f51a0 2023-11-17T05:30:42.977 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:42.975+0000 7f9b5af38700 1 -- 172.21.15.66:0/1307378755 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f9b54106ba0 con 0x7f9b540f51a0 2023-11-17T05:30:42.977 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:42.975+0000 7f9b467fc700 1 -- 172.21.15.66:0/1307378755 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(25 keys) v1 ==== 1074+0+0 (secure 0 0 0) 0x7f9b50013f00 con 0x7f9b540f51a0 2023-11-17T05:30:42.977 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:42.975+0000 7f9b467fc700 1 -- 172.21.15.66:0/1307378755 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f9b50014dc0 con 0x7f9b540f51a0 2023-11-17T05:30:42.978 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:42.975+0000 7f9b5af38700 1 -- 172.21.15.66:0/1307378755 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f9b54107030 con 0x7f9b540f51a0 2023-11-17T05:30:42.978 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:42.976+0000 7f9b467fc700 1 -- 172.21.15.66:0/1307378755 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 17) v1 ==== 36269+0+0 (secure 0 0 0) 0x7f9b500095f0 con 0x7f9b540f51a0 2023-11-17T05:30:42.978 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:42.976+0000 7f9b5af38700 1 -- 172.21.15.66:0/1307378755 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f9b54045ad0 con 0x7f9b540f51a0 2023-11-17T05:30:42.978 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:42.976+0000 7f9b467fc700 1 --2- 172.21.15.66:0/1307378755 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f9b4002ff60 0x7f9b40032420 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:30:42.978 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:42.976+0000 7f9b467fc700 1 -- 172.21.15.66:0/1307378755 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(5..5 src has 1..5) v4 ==== 1270+0+0 (secure 0 0 0) 0x7f9b500410c0 con 0x7f9b540f51a0 2023-11-17T05:30:42.978 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:42.976+0000 7f9b59735700 1 --2- 172.21.15.66:0/1307378755 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f9b4002ff60 0x7f9b40032420 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:30:42.978 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:42.977+0000 7f9b59735700 1 --2- 172.21.15.66:0/1307378755 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f9b4002ff60 0x7f9b40032420 secure :-1 s=READY pgs=116 cs=0 l=1 rev1=1 rx=0x7f9b480096a0 tx=0x7f9b48006b40).ready entity=mgr.14164 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:30:42.981 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:42.981+0000 7f9b467fc700 1 -- 172.21.15.66:0/1307378755 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+152212 (secure 0 0 0) 0x7f9b50014420 con 0x7f9b540f51a0 2023-11-17T05:30:42.991 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:30:42 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:30:41.251279+0000 mgr.smithi040.ogcprl (mgr.14164) 293 : cluster [DBG] pgmap v279: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:30:43.201 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:43.201+0000 7f9b5af38700 1 -- 172.21.15.66:0/1307378755 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "mon dump", "format": "json"} v 0) v1 -- 0x7f9b54107ee0 con 0x7f9b540f51a0 2023-11-17T05:30:43.202 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:43.202+0000 7f9b467fc700 1 -- 172.21.15.66:0/1307378755 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mon_command_ack([{"prefix": "mon dump", "format": "json"}]=0 dumped monmap epoch 1 v1) v1 ==== 95+0+690 (secure 0 0 0) 0x7f9b50010230 con 0x7f9b540f51a0 2023-11-17T05:30:43.202 INFO:teuthology.orchestra.run.smithi066.stdout: 2023-11-17T05:30:43.202 INFO:teuthology.orchestra.run.smithi066.stdout:{"epoch":1,"fsid":"ed6470e2-8508-11ee-8dbe-212e2dc638e7","modified":"2023-11-17T05:20:13.934038Z","created":"2023-11-17T05:20:13.934038Z","min_mon_release":16,"min_mon_release_name":"pacific","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging"],"optional":[]},"mons":[{"rank":0,"name":"smithi040","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:3300","nonce":0},{"type":"v1","addr":"172.21.15.40:6789","nonce":0}]},"addr":"172.21.15.40:6789/0","public_addr":"172.21.15.40:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2023-11-17T05:30:43.204 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:43.203+0000 7f9b5af38700 1 -- 172.21.15.66:0/1307378755 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f9b4002ff60 msgr2=0x7f9b40032420 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:30:43.204 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:43.203+0000 7f9b5af38700 1 --2- 172.21.15.66:0/1307378755 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f9b4002ff60 0x7f9b40032420 secure :-1 s=READY pgs=116 cs=0 l=1 rev1=1 rx=0x7f9b480096a0 tx=0x7f9b48006b40).stop 2023-11-17T05:30:43.204 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:43.203+0000 7f9b5af38700 1 -- 172.21.15.66:0/1307378755 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f9b540f51a0 msgr2=0x7f9b541064b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:30:43.204 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:43.203+0000 7f9b5af38700 1 --2- 172.21.15.66:0/1307378755 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f9b540f51a0 0x7f9b541064b0 secure :-1 s=READY pgs=302 cs=0 l=1 rev1=1 rx=0x7f9b50013510 tx=0x7f9b5000b040).stop 2023-11-17T05:30:43.204 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:43.203+0000 7f9b5af38700 1 -- 172.21.15.66:0/1307378755 shutdown_connections 2023-11-17T05:30:43.205 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:43.203+0000 7f9b5af38700 1 --2- 172.21.15.66:0/1307378755 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f9b4002ff60 0x7f9b40032420 unknown :-1 s=CLOSED pgs=116 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:30:43.205 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:43.203+0000 7f9b5af38700 1 --2- 172.21.15.66:0/1307378755 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f9b540f51a0 0x7f9b541064b0 unknown :-1 s=CLOSED pgs=302 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:30:43.205 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:43.203+0000 7f9b5af38700 1 -- 172.21.15.66:0/1307378755 >> 172.21.15.66:0/1307378755 conn(0x7f9b540f09a0 msgr2=0x7f9b540f1400 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:30:43.205 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:43.203+0000 7f9b5af38700 1 -- 172.21.15.66:0/1307378755 shutdown_connections 2023-11-17T05:30:43.205 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:43.203+0000 7f9b5af38700 1 -- 172.21.15.66:0/1307378755 wait complete. 2023-11-17T05:30:43.206 INFO:teuthology.orchestra.run.smithi066.stderr:dumped monmap epoch 1 2023-11-17T05:30:43.989 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:30:43 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:30:43.202472+0000 mon.smithi040 (mon.0) 251 : audit [DBG] from='client.? 172.21.15.66:0/1307378755' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-11-17T05:30:44.586 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2023-11-17T05:30:44.589 DEBUG:teuthology.orchestra.run.smithi066:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.4 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ed6470e2-8508-11ee-8dbe-212e2dc638e7 -- ceph mon dump -f json 2023-11-17T05:30:44.988 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:30:44 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:30:43.251588+0000 mgr.smithi040.ogcprl (mgr.14164) 294 : cluster [DBG] pgmap v280: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:30:45.853 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:45.852+0000 7f4f8e40e700 1 -- 172.21.15.66:0/3503749690 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f4f880f5410 msgr2=0x7f4f880f5830 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:30:45.853 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:45.852+0000 7f4f8e40e700 1 --2- 172.21.15.66:0/3503749690 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f4f880f5410 0x7f4f880f5830 secure :-1 s=READY pgs=303 cs=0 l=1 rev1=1 rx=0x7f4f78009fb0 tx=0x7f4f78009b30).stop 2023-11-17T05:30:45.853 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:45.852+0000 7f4f8e40e700 1 -- 172.21.15.66:0/3503749690 shutdown_connections 2023-11-17T05:30:45.853 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:45.852+0000 7f4f8e40e700 1 --2- 172.21.15.66:0/3503749690 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f4f880f5410 0x7f4f880f5830 unknown :-1 s=CLOSED pgs=303 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:30:45.853 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:45.852+0000 7f4f8e40e700 1 -- 172.21.15.66:0/3503749690 >> 172.21.15.66:0/3503749690 conn(0x7f4f880f0b70 msgr2=0x7f4f880f2f90 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:30:45.853 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:45.852+0000 7f4f8e40e700 1 -- 172.21.15.66:0/3503749690 shutdown_connections 2023-11-17T05:30:45.853 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:45.852+0000 7f4f8e40e700 1 -- 172.21.15.66:0/3503749690 wait complete. 2023-11-17T05:30:45.853 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:45.853+0000 7f4f8e40e700 1 Processor -- start 2023-11-17T05:30:45.854 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:45.853+0000 7f4f8e40e700 1 -- start start 2023-11-17T05:30:45.855 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:45.853+0000 7f4f8e40e700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f4f880f5410 0x7f4f88109450 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:30:45.855 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:45.853+0000 7f4f8e40e700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f4f88060220 con 0x7f4f880f5410 2023-11-17T05:30:45.855 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:45.853+0000 7f4f87fff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f4f880f5410 0x7f4f88109450 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:30:45.855 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:45.853+0000 7f4f87fff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f4f880f5410 0x7f4f88109450 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.66:39934/0 (socket says 172.21.15.66:39934) 2023-11-17T05:30:45.855 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:45.853+0000 7f4f87fff700 1 -- 172.21.15.66:0/2849158045 learned_addr learned my addr 172.21.15.66:0/2849158045 (peer_addr_for_me v2:172.21.15.66:0/0) 2023-11-17T05:30:45.855 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:45.854+0000 7f4f87fff700 1 -- 172.21.15.66:0/2849158045 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f4f78005040 con 0x7f4f880f5410 2023-11-17T05:30:45.855 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:45.854+0000 7f4f87fff700 1 --2- 172.21.15.66:0/2849158045 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f4f880f5410 0x7f4f88109450 secure :-1 s=READY pgs=304 cs=0 l=1 rev1=1 rx=0x7f4f78013510 tx=0x7f4f7800b040).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:30:45.855 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:45.854+0000 7f4f84ff9700 1 -- 172.21.15.66:0/2849158045 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f4f78009320 con 0x7f4f880f5410 2023-11-17T05:30:45.855 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:45.854+0000 7f4f8e40e700 1 -- 172.21.15.66:0/2849158045 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f4f88060420 con 0x7f4f880f5410 2023-11-17T05:30:45.855 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:45.854+0000 7f4f84ff9700 1 -- 172.21.15.66:0/2849158045 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(25 keys) v1 ==== 1074+0+0 (secure 0 0 0) 0x7f4f78013f00 con 0x7f4f880f5410 2023-11-17T05:30:45.855 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:45.854+0000 7f4f8e40e700 1 -- 172.21.15.66:0/2849158045 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f4f88109b00 con 0x7f4f880f5410 2023-11-17T05:30:45.855 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:45.854+0000 7f4f84ff9700 1 -- 172.21.15.66:0/2849158045 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f4f78014d90 con 0x7f4f880f5410 2023-11-17T05:30:45.856 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:45.855+0000 7f4f84ff9700 1 -- 172.21.15.66:0/2849158045 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 17) v1 ==== 36269+0+0 (secure 0 0 0) 0x7f4f78014420 con 0x7f4f880f5410 2023-11-17T05:30:45.856 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:45.855+0000 7f4f8e40e700 1 -- 172.21.15.66:0/2849158045 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f4f88059100 con 0x7f4f880f5410 2023-11-17T05:30:45.856 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:45.855+0000 7f4f84ff9700 1 --2- 172.21.15.66:0/2849158045 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f4f7002ffb0 0x7f4f70032470 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:30:45.856 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:45.855+0000 7f4f84ff9700 1 -- 172.21.15.66:0/2849158045 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(5..5 src has 1..5) v4 ==== 1270+0+0 (secure 0 0 0) 0x7f4f78041100 con 0x7f4f880f5410 2023-11-17T05:30:45.856 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:45.855+0000 7f4f877fe700 1 --2- 172.21.15.66:0/2849158045 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f4f7002ffb0 0x7f4f70032470 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:30:45.857 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:45.856+0000 7f4f877fe700 1 --2- 172.21.15.66:0/2849158045 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f4f7002ffb0 0x7f4f70032470 secure :-1 s=READY pgs=117 cs=0 l=1 rev1=1 rx=0x7f4f7c0096a0 tx=0x7f4f7c006b40).ready entity=mgr.14164 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:30:45.860 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:45.860+0000 7f4f84ff9700 1 -- 172.21.15.66:0/2849158045 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+152212 (secure 0 0 0) 0x7f4f780095f0 con 0x7f4f880f5410 2023-11-17T05:30:46.080 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:46.079+0000 7f4f8e40e700 1 -- 172.21.15.66:0/2849158045 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "mon dump", "format": "json"} v 0) v1 -- 0x7f4f88045ad0 con 0x7f4f880f5410 2023-11-17T05:30:46.080 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:46.080+0000 7f4f84ff9700 1 -- 172.21.15.66:0/2849158045 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mon_command_ack([{"prefix": "mon dump", "format": "json"}]=0 dumped monmap epoch 1 v1) v1 ==== 95+0+690 (secure 0 0 0) 0x7f4f7801db70 con 0x7f4f880f5410 2023-11-17T05:30:46.081 INFO:teuthology.orchestra.run.smithi066.stdout: 2023-11-17T05:30:46.081 INFO:teuthology.orchestra.run.smithi066.stdout:{"epoch":1,"fsid":"ed6470e2-8508-11ee-8dbe-212e2dc638e7","modified":"2023-11-17T05:20:13.934038Z","created":"2023-11-17T05:20:13.934038Z","min_mon_release":16,"min_mon_release_name":"pacific","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging"],"optional":[]},"mons":[{"rank":0,"name":"smithi040","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:3300","nonce":0},{"type":"v1","addr":"172.21.15.40:6789","nonce":0}]},"addr":"172.21.15.40:6789/0","public_addr":"172.21.15.40:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2023-11-17T05:30:46.082 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:46.082+0000 7f4f8e40e700 1 -- 172.21.15.66:0/2849158045 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f4f7002ffb0 msgr2=0x7f4f70032470 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:30:46.082 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:46.082+0000 7f4f8e40e700 1 --2- 172.21.15.66:0/2849158045 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f4f7002ffb0 0x7f4f70032470 secure :-1 s=READY pgs=117 cs=0 l=1 rev1=1 rx=0x7f4f7c0096a0 tx=0x7f4f7c006b40).stop 2023-11-17T05:30:46.082 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:46.082+0000 7f4f8e40e700 1 -- 172.21.15.66:0/2849158045 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f4f880f5410 msgr2=0x7f4f88109450 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:30:46.082 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:46.082+0000 7f4f8e40e700 1 --2- 172.21.15.66:0/2849158045 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f4f880f5410 0x7f4f88109450 secure :-1 s=READY pgs=304 cs=0 l=1 rev1=1 rx=0x7f4f78013510 tx=0x7f4f7800b040).stop 2023-11-17T05:30:46.082 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:46.082+0000 7f4f8e40e700 1 -- 172.21.15.66:0/2849158045 shutdown_connections 2023-11-17T05:30:46.082 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:46.082+0000 7f4f8e40e700 1 --2- 172.21.15.66:0/2849158045 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f4f7002ffb0 0x7f4f70032470 unknown :-1 s=CLOSED pgs=117 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:30:46.083 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:46.082+0000 7f4f8e40e700 1 --2- 172.21.15.66:0/2849158045 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f4f880f5410 0x7f4f88109450 unknown :-1 s=CLOSED pgs=304 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:30:46.083 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:46.082+0000 7f4f8e40e700 1 -- 172.21.15.66:0/2849158045 >> 172.21.15.66:0/2849158045 conn(0x7f4f880f0b70 msgr2=0x7f4f880f1660 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:30:46.083 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:46.082+0000 7f4f8e40e700 1 -- 172.21.15.66:0/2849158045 shutdown_connections 2023-11-17T05:30:46.083 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:46.082+0000 7f4f8e40e700 1 -- 172.21.15.66:0/2849158045 wait complete. 2023-11-17T05:30:46.083 INFO:teuthology.orchestra.run.smithi066.stderr:dumped monmap epoch 1 2023-11-17T05:30:46.988 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:30:46 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:30:45.251826+0000 mgr.smithi040.ogcprl (mgr.14164) 295 : cluster [DBG] pgmap v281: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:30:46.989 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:30:46 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:30:46.080946+0000 mon.smithi040 (mon.0) 252 : audit [DBG] from='client.? 172.21.15.66:0/2849158045' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-11-17T05:30:47.462 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2023-11-17T05:30:47.462 DEBUG:teuthology.orchestra.run.smithi066:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.4 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ed6470e2-8508-11ee-8dbe-212e2dc638e7 -- ceph mon dump -f json 2023-11-17T05:30:48.553 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:48.551+0000 7feabb2b4700 1 -- 172.21.15.66:0/106092252 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7feab4060280 msgr2=0x7feab40f90f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:30:48.553 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:48.551+0000 7feabb2b4700 1 --2- 172.21.15.66:0/106092252 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7feab4060280 0x7feab40f90f0 secure :-1 s=READY pgs=305 cs=0 l=1 rev1=1 rx=0x7feab0009fb0 tx=0x7feab0009b30).stop 2023-11-17T05:30:48.553 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:48.551+0000 7feabb2b4700 1 -- 172.21.15.66:0/106092252 shutdown_connections 2023-11-17T05:30:48.553 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:48.551+0000 7feabb2b4700 1 --2- 172.21.15.66:0/106092252 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7feab4060280 0x7feab40f90f0 unknown :-1 s=CLOSED pgs=305 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:30:48.553 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:48.551+0000 7feabb2b4700 1 -- 172.21.15.66:0/106092252 >> 172.21.15.66:0/106092252 conn(0x7feab40f0b70 msgr2=0x7feab40f2f90 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:30:48.553 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:48.552+0000 7feabb2b4700 1 -- 172.21.15.66:0/106092252 shutdown_connections 2023-11-17T05:30:48.553 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:48.552+0000 7feabb2b4700 1 -- 172.21.15.66:0/106092252 wait complete. 2023-11-17T05:30:48.553 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:48.552+0000 7feabb2b4700 1 Processor -- start 2023-11-17T05:30:48.553 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:48.552+0000 7feabb2b4700 1 -- start start 2023-11-17T05:30:48.554 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:48.552+0000 7feabb2b4700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7feab4060280 0x7feab40f6b70 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:30:48.554 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:48.552+0000 7feabb2b4700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7feab40f4ff0 con 0x7feab4060280 2023-11-17T05:30:48.554 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:48.552+0000 7feab9050700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7feab4060280 0x7feab40f6b70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:30:48.554 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:48.553+0000 7feab9050700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7feab4060280 0x7feab40f6b70 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.66:39964/0 (socket says 172.21.15.66:39964) 2023-11-17T05:30:48.554 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:48.553+0000 7feab9050700 1 -- 172.21.15.66:0/3696301966 learned_addr learned my addr 172.21.15.66:0/3696301966 (peer_addr_for_me v2:172.21.15.66:0/0) 2023-11-17T05:30:48.554 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:48.553+0000 7feab9050700 1 -- 172.21.15.66:0/3696301966 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7feab0005040 con 0x7feab4060280 2023-11-17T05:30:48.555 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:48.553+0000 7feab9050700 1 --2- 172.21.15.66:0/3696301966 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7feab4060280 0x7feab40f6b70 secure :-1 s=READY pgs=306 cs=0 l=1 rev1=1 rx=0x7feab0013510 tx=0x7feab000b040).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:30:48.555 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:48.553+0000 7feaa9ffb700 1 -- 172.21.15.66:0/3696301966 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7feab0009320 con 0x7feab4060280 2023-11-17T05:30:48.555 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:48.553+0000 7feabb2b4700 1 -- 172.21.15.66:0/3696301966 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7feab40f5190 con 0x7feab4060280 2023-11-17T05:30:48.556 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:48.553+0000 7feaa9ffb700 1 -- 172.21.15.66:0/3696301966 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(25 keys) v1 ==== 1074+0+0 (secure 0 0 0) 0x7feab0013f00 con 0x7feab4060280 2023-11-17T05:30:48.556 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:48.553+0000 7feaa9ffb700 1 -- 172.21.15.66:0/3696301966 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7feab0014dc0 con 0x7feab4060280 2023-11-17T05:30:48.556 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:48.553+0000 7feabb2b4700 1 -- 172.21.15.66:0/3696301966 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7feab40f5620 con 0x7feab4060280 2023-11-17T05:30:48.556 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:48.554+0000 7feaa9ffb700 1 -- 172.21.15.66:0/3696301966 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 17) v1 ==== 36269+0+0 (secure 0 0 0) 0x7feab00095f0 con 0x7feab4060280 2023-11-17T05:30:48.556 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:48.554+0000 7feabb2b4700 1 -- 172.21.15.66:0/3696301966 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7feab4059100 con 0x7feab4060280 2023-11-17T05:30:48.556 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:48.554+0000 7feaa9ffb700 1 --2- 172.21.15.66:0/3696301966 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7feaa002ff60 0x7feaa0032420 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:30:48.556 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:48.554+0000 7feaa9ffb700 1 -- 172.21.15.66:0/3696301966 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(5..5 src has 1..5) v4 ==== 1270+0+0 (secure 0 0 0) 0x7feab0041100 con 0x7feab4060280 2023-11-17T05:30:48.556 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:48.554+0000 7feab884f700 1 --2- 172.21.15.66:0/3696301966 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7feaa002ff60 0x7feaa0032420 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:30:48.556 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:48.555+0000 7feab884f700 1 --2- 172.21.15.66:0/3696301966 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7feaa002ff60 0x7feaa0032420 secure :-1 s=READY pgs=118 cs=0 l=1 rev1=1 rx=0x7feaa40096a0 tx=0x7feaa4006b40).ready entity=mgr.14164 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:30:48.559 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:48.559+0000 7feaa9ffb700 1 -- 172.21.15.66:0/3696301966 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+152212 (secure 0 0 0) 0x7feab0014420 con 0x7feab4060280 2023-11-17T05:30:48.778 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:48.778+0000 7feabb2b4700 1 -- 172.21.15.66:0/3696301966 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "mon dump", "format": "json"} v 0) v1 -- 0x7feab40fbf50 con 0x7feab4060280 2023-11-17T05:30:48.780 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:48.779+0000 7feaa9ffb700 1 -- 172.21.15.66:0/3696301966 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mon_command_ack([{"prefix": "mon dump", "format": "json"}]=0 dumped monmap epoch 1 v1) v1 ==== 95+0+690 (secure 0 0 0) 0x7feab0010230 con 0x7feab4060280 2023-11-17T05:30:48.781 INFO:teuthology.orchestra.run.smithi066.stdout: 2023-11-17T05:30:48.781 INFO:teuthology.orchestra.run.smithi066.stdout:{"epoch":1,"fsid":"ed6470e2-8508-11ee-8dbe-212e2dc638e7","modified":"2023-11-17T05:20:13.934038Z","created":"2023-11-17T05:20:13.934038Z","min_mon_release":16,"min_mon_release_name":"pacific","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging"],"optional":[]},"mons":[{"rank":0,"name":"smithi040","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:3300","nonce":0},{"type":"v1","addr":"172.21.15.40:6789","nonce":0}]},"addr":"172.21.15.40:6789/0","public_addr":"172.21.15.40:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2023-11-17T05:30:48.781 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:48.780+0000 7feabb2b4700 1 -- 172.21.15.66:0/3696301966 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7feaa002ff60 msgr2=0x7feaa0032420 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:30:48.782 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:48.780+0000 7feabb2b4700 1 --2- 172.21.15.66:0/3696301966 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7feaa002ff60 0x7feaa0032420 secure :-1 s=READY pgs=118 cs=0 l=1 rev1=1 rx=0x7feaa40096a0 tx=0x7feaa4006b40).stop 2023-11-17T05:30:48.782 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:48.780+0000 7feabb2b4700 1 -- 172.21.15.66:0/3696301966 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7feab4060280 msgr2=0x7feab40f6b70 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:30:48.782 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:48.780+0000 7feabb2b4700 1 --2- 172.21.15.66:0/3696301966 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7feab4060280 0x7feab40f6b70 secure :-1 s=READY pgs=306 cs=0 l=1 rev1=1 rx=0x7feab0013510 tx=0x7feab000b040).stop 2023-11-17T05:30:48.782 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:48.780+0000 7feabb2b4700 1 -- 172.21.15.66:0/3696301966 shutdown_connections 2023-11-17T05:30:48.782 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:48.780+0000 7feabb2b4700 1 --2- 172.21.15.66:0/3696301966 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7feaa002ff60 0x7feaa0032420 unknown :-1 s=CLOSED pgs=118 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:30:48.782 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:48.780+0000 7feabb2b4700 1 --2- 172.21.15.66:0/3696301966 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7feab4060280 0x7feab40f6b70 unknown :-1 s=CLOSED pgs=306 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:30:48.782 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:48.780+0000 7feabb2b4700 1 -- 172.21.15.66:0/3696301966 >> 172.21.15.66:0/3696301966 conn(0x7feab40f0b70 msgr2=0x7feab40f1660 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:30:48.782 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:48.781+0000 7feabb2b4700 1 -- 172.21.15.66:0/3696301966 shutdown_connections 2023-11-17T05:30:48.782 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:48.781+0000 7feabb2b4700 1 -- 172.21.15.66:0/3696301966 wait complete. 2023-11-17T05:30:48.782 INFO:teuthology.orchestra.run.smithi066.stderr:dumped monmap epoch 1 2023-11-17T05:30:48.989 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:30:48 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:30:47.252175+0000 mgr.smithi040.ogcprl (mgr.14164) 296 : cluster [DBG] pgmap v282: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:30:49.988 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:30:49 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:30:48.779291+0000 mon.smithi040 (mon.0) 253 : audit [DBG] from='client.? 172.21.15.66:0/3696301966' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-11-17T05:30:50.181 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2023-11-17T05:30:50.181 DEBUG:teuthology.orchestra.run.smithi066:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.4 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ed6470e2-8508-11ee-8dbe-212e2dc638e7 -- ceph mon dump -f json 2023-11-17T05:30:50.989 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:30:50 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:30:49.252480+0000 mgr.smithi040.ogcprl (mgr.14164) 297 : cluster [DBG] pgmap v283: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:30:51.385 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:51.385+0000 7f8f57472700 1 -- 172.21.15.66:0/1064101174 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f8f50060280 msgr2=0x7f8f500f90f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:30:51.386 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:51.385+0000 7f8f57472700 1 --2- 172.21.15.66:0/1064101174 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f8f50060280 0x7f8f500f90f0 secure :-1 s=READY pgs=307 cs=0 l=1 rev1=1 rx=0x7f8f4c009fb0 tx=0x7f8f4c009b30).stop 2023-11-17T05:30:51.386 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:51.385+0000 7f8f57472700 1 -- 172.21.15.66:0/1064101174 shutdown_connections 2023-11-17T05:30:51.386 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:51.385+0000 7f8f57472700 1 --2- 172.21.15.66:0/1064101174 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f8f50060280 0x7f8f500f90f0 unknown :-1 s=CLOSED pgs=307 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:30:51.386 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:51.385+0000 7f8f57472700 1 -- 172.21.15.66:0/1064101174 >> 172.21.15.66:0/1064101174 conn(0x7f8f500f0b70 msgr2=0x7f8f500f2f90 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:30:51.386 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:51.385+0000 7f8f57472700 1 -- 172.21.15.66:0/1064101174 shutdown_connections 2023-11-17T05:30:51.386 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:51.385+0000 7f8f57472700 1 -- 172.21.15.66:0/1064101174 wait complete. 2023-11-17T05:30:51.387 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:51.386+0000 7f8f57472700 1 Processor -- start 2023-11-17T05:30:51.387 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:51.386+0000 7f8f57472700 1 -- start start 2023-11-17T05:30:51.388 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:51.386+0000 7f8f57472700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f8f50060280 0x7f8f501088f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:30:51.388 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:51.386+0000 7f8f57472700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f8f50108de0 con 0x7f8f50060280 2023-11-17T05:30:51.388 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:51.386+0000 7f8f5520e700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f8f50060280 0x7f8f501088f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:30:51.388 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:51.386+0000 7f8f5520e700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f8f50060280 0x7f8f501088f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.66:39996/0 (socket says 172.21.15.66:39996) 2023-11-17T05:30:51.388 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:51.386+0000 7f8f5520e700 1 -- 172.21.15.66:0/4068947184 learned_addr learned my addr 172.21.15.66:0/4068947184 (peer_addr_for_me v2:172.21.15.66:0/0) 2023-11-17T05:30:51.388 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:51.386+0000 7f8f5520e700 1 -- 172.21.15.66:0/4068947184 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f8f4c005040 con 0x7f8f50060280 2023-11-17T05:30:51.388 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:51.386+0000 7f8f5520e700 1 --2- 172.21.15.66:0/4068947184 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f8f50060280 0x7f8f501088f0 secure :-1 s=READY pgs=308 cs=0 l=1 rev1=1 rx=0x7f8f4c013510 tx=0x7f8f4c00b040).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:30:51.388 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:51.387+0000 7f8f45ffb700 1 -- 172.21.15.66:0/4068947184 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f8f4c009320 con 0x7f8f50060280 2023-11-17T05:30:51.388 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:51.387+0000 7f8f57472700 1 -- 172.21.15.66:0/4068947184 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f8f50108fe0 con 0x7f8f50060280 2023-11-17T05:30:51.388 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:51.387+0000 7f8f45ffb700 1 -- 172.21.15.66:0/4068947184 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(25 keys) v1 ==== 1074+0+0 (secure 0 0 0) 0x7f8f4c013f00 con 0x7f8f50060280 2023-11-17T05:30:51.388 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:51.387+0000 7f8f45ffb700 1 -- 172.21.15.66:0/4068947184 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f8f4c014dc0 con 0x7f8f50060280 2023-11-17T05:30:51.388 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:51.387+0000 7f8f57472700 1 -- 172.21.15.66:0/4068947184 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f8f50109470 con 0x7f8f50060280 2023-11-17T05:30:51.389 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:51.388+0000 7f8f45ffb700 1 -- 172.21.15.66:0/4068947184 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 17) v1 ==== 36269+0+0 (secure 0 0 0) 0x7f8f4c0095f0 con 0x7f8f50060280 2023-11-17T05:30:51.389 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:51.388+0000 7f8f57472700 1 -- 172.21.15.66:0/4068947184 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f8f50045ad0 con 0x7f8f50060280 2023-11-17T05:30:51.389 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:51.388+0000 7f8f45ffb700 1 --2- 172.21.15.66:0/4068947184 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f8f3c02ff60 0x7f8f3c032420 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:30:51.389 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:51.388+0000 7f8f45ffb700 1 -- 172.21.15.66:0/4068947184 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(5..5 src has 1..5) v4 ==== 1270+0+0 (secure 0 0 0) 0x7f8f4c041020 con 0x7f8f50060280 2023-11-17T05:30:51.390 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:51.388+0000 7f8f54a0d700 1 --2- 172.21.15.66:0/4068947184 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f8f3c02ff60 0x7f8f3c032420 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:30:51.390 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:51.389+0000 7f8f54a0d700 1 --2- 172.21.15.66:0/4068947184 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f8f3c02ff60 0x7f8f3c032420 secure :-1 s=READY pgs=119 cs=0 l=1 rev1=1 rx=0x7f8f400096a0 tx=0x7f8f40006b40).ready entity=mgr.14164 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:30:51.393 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:51.393+0000 7f8f45ffb700 1 -- 172.21.15.66:0/4068947184 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+152212 (secure 0 0 0) 0x7f8f4c014420 con 0x7f8f50060280 2023-11-17T05:30:51.612 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:51.612+0000 7f8f57472700 1 -- 172.21.15.66:0/4068947184 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "mon dump", "format": "json"} v 0) v1 -- 0x7f8f500fbf50 con 0x7f8f50060280 2023-11-17T05:30:51.613 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:51.613+0000 7f8f45ffb700 1 -- 172.21.15.66:0/4068947184 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mon_command_ack([{"prefix": "mon dump", "format": "json"}]=0 dumped monmap epoch 1 v1) v1 ==== 95+0+690 (secure 0 0 0) 0x7f8f4c01c920 con 0x7f8f50060280 2023-11-17T05:30:51.613 INFO:teuthology.orchestra.run.smithi066.stdout: 2023-11-17T05:30:51.613 INFO:teuthology.orchestra.run.smithi066.stdout:{"epoch":1,"fsid":"ed6470e2-8508-11ee-8dbe-212e2dc638e7","modified":"2023-11-17T05:20:13.934038Z","created":"2023-11-17T05:20:13.934038Z","min_mon_release":16,"min_mon_release_name":"pacific","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging"],"optional":[]},"mons":[{"rank":0,"name":"smithi040","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:3300","nonce":0},{"type":"v1","addr":"172.21.15.40:6789","nonce":0}]},"addr":"172.21.15.40:6789/0","public_addr":"172.21.15.40:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2023-11-17T05:30:51.615 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:51.614+0000 7f8f57472700 1 -- 172.21.15.66:0/4068947184 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f8f3c02ff60 msgr2=0x7f8f3c032420 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:30:51.615 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:51.614+0000 7f8f57472700 1 --2- 172.21.15.66:0/4068947184 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f8f3c02ff60 0x7f8f3c032420 secure :-1 s=READY pgs=119 cs=0 l=1 rev1=1 rx=0x7f8f400096a0 tx=0x7f8f40006b40).stop 2023-11-17T05:30:51.615 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:51.614+0000 7f8f57472700 1 -- 172.21.15.66:0/4068947184 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f8f50060280 msgr2=0x7f8f501088f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:30:51.615 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:51.614+0000 7f8f57472700 1 --2- 172.21.15.66:0/4068947184 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f8f50060280 0x7f8f501088f0 secure :-1 s=READY pgs=308 cs=0 l=1 rev1=1 rx=0x7f8f4c013510 tx=0x7f8f4c00b040).stop 2023-11-17T05:30:51.615 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:51.614+0000 7f8f57472700 1 -- 172.21.15.66:0/4068947184 shutdown_connections 2023-11-17T05:30:51.615 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:51.614+0000 7f8f57472700 1 --2- 172.21.15.66:0/4068947184 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f8f3c02ff60 0x7f8f3c032420 unknown :-1 s=CLOSED pgs=119 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:30:51.615 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:51.614+0000 7f8f57472700 1 --2- 172.21.15.66:0/4068947184 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f8f50060280 0x7f8f501088f0 unknown :-1 s=CLOSED pgs=308 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:30:51.615 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:51.614+0000 7f8f57472700 1 -- 172.21.15.66:0/4068947184 >> 172.21.15.66:0/4068947184 conn(0x7f8f500f0b70 msgr2=0x7f8f500f1620 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:30:51.615 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:51.614+0000 7f8f57472700 1 -- 172.21.15.66:0/4068947184 shutdown_connections 2023-11-17T05:30:51.615 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:51.614+0000 7f8f57472700 1 -- 172.21.15.66:0/4068947184 wait complete. 2023-11-17T05:30:51.616 INFO:teuthology.orchestra.run.smithi066.stderr:dumped monmap epoch 1 2023-11-17T05:30:52.989 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:30:52 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:30:51.252811+0000 mgr.smithi040.ogcprl (mgr.14164) 298 : cluster [DBG] pgmap v284: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:30:52.989 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:30:52 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:30:51.613437+0000 mon.smithi040 (mon.0) 254 : audit [DBG] from='client.? 172.21.15.66:0/4068947184' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-11-17T05:30:53.002 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2023-11-17T05:30:53.003 DEBUG:teuthology.orchestra.run.smithi066:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.4 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ed6470e2-8508-11ee-8dbe-212e2dc638e7 -- ceph mon dump -f json 2023-11-17T05:30:54.215 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:54.212+0000 7fce7536a700 1 -- 172.21.15.66:0/3075156155 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fce700f6510 msgr2=0x7fce700f6930 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:30:54.215 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:54.212+0000 7fce7536a700 1 --2- 172.21.15.66:0/3075156155 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fce700f6510 0x7fce700f6930 secure :-1 s=READY pgs=309 cs=0 l=1 rev1=1 rx=0x7fce60009fb0 tx=0x7fce60009b30).stop 2023-11-17T05:30:54.215 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:54.212+0000 7fce7536a700 1 -- 172.21.15.66:0/3075156155 shutdown_connections 2023-11-17T05:30:54.215 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:54.212+0000 7fce7536a700 1 --2- 172.21.15.66:0/3075156155 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fce700f6510 0x7fce700f6930 unknown :-1 s=CLOSED pgs=309 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:30:54.215 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:54.212+0000 7fce7536a700 1 -- 172.21.15.66:0/3075156155 >> 172.21.15.66:0/3075156155 conn(0x7fce700f1c70 msgr2=0x7fce700f4090 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:30:54.215 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:54.212+0000 7fce7536a700 1 -- 172.21.15.66:0/3075156155 shutdown_connections 2023-11-17T05:30:54.215 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:54.212+0000 7fce7536a700 1 -- 172.21.15.66:0/3075156155 wait complete. 2023-11-17T05:30:54.216 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:54.213+0000 7fce7536a700 1 Processor -- start 2023-11-17T05:30:54.216 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:54.213+0000 7fce7536a700 1 -- start start 2023-11-17T05:30:54.216 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:54.213+0000 7fce7536a700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fce700f6510 0x7fce7006b9a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:30:54.216 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:54.213+0000 7fce7536a700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fce70069e20 con 0x7fce700f6510 2023-11-17T05:30:54.216 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:54.214+0000 7fce6effd700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fce700f6510 0x7fce7006b9a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:30:54.216 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:54.214+0000 7fce6effd700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fce700f6510 0x7fce7006b9a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.66:37196/0 (socket says 172.21.15.66:37196) 2023-11-17T05:30:54.216 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:54.214+0000 7fce6effd700 1 -- 172.21.15.66:0/2168050709 learned_addr learned my addr 172.21.15.66:0/2168050709 (peer_addr_for_me v2:172.21.15.66:0/0) 2023-11-17T05:30:54.216 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:54.214+0000 7fce6effd700 1 -- 172.21.15.66:0/2168050709 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fce60005040 con 0x7fce700f6510 2023-11-17T05:30:54.216 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:54.214+0000 7fce6effd700 1 --2- 172.21.15.66:0/2168050709 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fce700f6510 0x7fce7006b9a0 secure :-1 s=READY pgs=310 cs=0 l=1 rev1=1 rx=0x7fce60013510 tx=0x7fce6000b040).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:30:54.217 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:54.214+0000 7fce577fe700 1 -- 172.21.15.66:0/2168050709 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7fce60009320 con 0x7fce700f6510 2023-11-17T05:30:54.217 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:54.214+0000 7fce7536a700 1 -- 172.21.15.66:0/2168050709 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fce70069fc0 con 0x7fce700f6510 2023-11-17T05:30:54.218 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:54.214+0000 7fce577fe700 1 -- 172.21.15.66:0/2168050709 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(25 keys) v1 ==== 1074+0+0 (secure 0 0 0) 0x7fce60013f00 con 0x7fce700f6510 2023-11-17T05:30:54.218 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:54.214+0000 7fce577fe700 1 -- 172.21.15.66:0/2168050709 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7fce60014d90 con 0x7fce700f6510 2023-11-17T05:30:54.218 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:54.214+0000 7fce7536a700 1 -- 172.21.15.66:0/2168050709 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fce7006a450 con 0x7fce700f6510 2023-11-17T05:30:54.218 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:54.215+0000 7fce7536a700 1 -- 172.21.15.66:0/2168050709 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fce70059100 con 0x7fce700f6510 2023-11-17T05:30:54.218 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:54.215+0000 7fce577fe700 1 -- 172.21.15.66:0/2168050709 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 17) v1 ==== 36269+0+0 (secure 0 0 0) 0x7fce600263a0 con 0x7fce700f6510 2023-11-17T05:30:54.218 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:54.215+0000 7fce577fe700 1 --2- 172.21.15.66:0/2168050709 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fce5802ff60 0x7fce58032420 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:30:54.218 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:54.215+0000 7fce577fe700 1 -- 172.21.15.66:0/2168050709 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(5..5 src has 1..5) v4 ==== 1270+0+0 (secure 0 0 0) 0x7fce60040fe0 con 0x7fce700f6510 2023-11-17T05:30:54.218 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:54.216+0000 7fce6e7fc700 1 --2- 172.21.15.66:0/2168050709 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fce5802ff60 0x7fce58032420 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:30:54.218 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:54.216+0000 7fce6e7fc700 1 --2- 172.21.15.66:0/2168050709 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fce5802ff60 0x7fce58032420 secure :-1 s=READY pgs=120 cs=0 l=1 rev1=1 rx=0x7fce640096a0 tx=0x7fce64006b40).ready entity=mgr.14164 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:30:54.221 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:54.220+0000 7fce577fe700 1 -- 172.21.15.66:0/2168050709 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+152212 (secure 0 0 0) 0x7fce600126f0 con 0x7fce700f6510 2023-11-17T05:30:54.440 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:54.439+0000 7fce7536a700 1 -- 172.21.15.66:0/2168050709 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "mon dump", "format": "json"} v 0) v1 -- 0x7fce70045ad0 con 0x7fce700f6510 2023-11-17T05:30:54.441 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:54.439+0000 7fce577fe700 1 -- 172.21.15.66:0/2168050709 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mon_command_ack([{"prefix": "mon dump", "format": "json"}]=0 dumped monmap epoch 1 v1) v1 ==== 95+0+690 (secure 0 0 0) 0x7fce6000f340 con 0x7fce700f6510 2023-11-17T05:30:54.441 INFO:teuthology.orchestra.run.smithi066.stdout: 2023-11-17T05:30:54.441 INFO:teuthology.orchestra.run.smithi066.stdout:{"epoch":1,"fsid":"ed6470e2-8508-11ee-8dbe-212e2dc638e7","modified":"2023-11-17T05:20:13.934038Z","created":"2023-11-17T05:20:13.934038Z","min_mon_release":16,"min_mon_release_name":"pacific","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging"],"optional":[]},"mons":[{"rank":0,"name":"smithi040","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:3300","nonce":0},{"type":"v1","addr":"172.21.15.40:6789","nonce":0}]},"addr":"172.21.15.40:6789/0","public_addr":"172.21.15.40:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2023-11-17T05:30:54.443 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:54.441+0000 7fce7536a700 1 -- 172.21.15.66:0/2168050709 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fce5802ff60 msgr2=0x7fce58032420 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:30:54.443 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:54.441+0000 7fce7536a700 1 --2- 172.21.15.66:0/2168050709 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fce5802ff60 0x7fce58032420 secure :-1 s=READY pgs=120 cs=0 l=1 rev1=1 rx=0x7fce640096a0 tx=0x7fce64006b40).stop 2023-11-17T05:30:54.443 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:54.441+0000 7fce7536a700 1 -- 172.21.15.66:0/2168050709 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fce700f6510 msgr2=0x7fce7006b9a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:30:54.443 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:54.441+0000 7fce7536a700 1 --2- 172.21.15.66:0/2168050709 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fce700f6510 0x7fce7006b9a0 secure :-1 s=READY pgs=310 cs=0 l=1 rev1=1 rx=0x7fce60013510 tx=0x7fce6000b040).stop 2023-11-17T05:30:54.443 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:54.441+0000 7fce7536a700 1 -- 172.21.15.66:0/2168050709 shutdown_connections 2023-11-17T05:30:54.444 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:54.441+0000 7fce7536a700 1 --2- 172.21.15.66:0/2168050709 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fce5802ff60 0x7fce58032420 unknown :-1 s=CLOSED pgs=120 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:30:54.444 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:54.441+0000 7fce7536a700 1 --2- 172.21.15.66:0/2168050709 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fce700f6510 0x7fce7006b9a0 unknown :-1 s=CLOSED pgs=310 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:30:54.444 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:54.441+0000 7fce7536a700 1 -- 172.21.15.66:0/2168050709 >> 172.21.15.66:0/2168050709 conn(0x7fce700f1c70 msgr2=0x7fce700f2760 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:30:54.444 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:54.441+0000 7fce7536a700 1 -- 172.21.15.66:0/2168050709 shutdown_connections 2023-11-17T05:30:54.444 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:54.441+0000 7fce7536a700 1 -- 172.21.15.66:0/2168050709 wait complete. 2023-11-17T05:30:54.444 INFO:teuthology.orchestra.run.smithi066.stderr:dumped monmap epoch 1 2023-11-17T05:30:54.989 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:30:54 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:30:53.253236+0000 mgr.smithi040.ogcprl (mgr.14164) 299 : cluster [DBG] pgmap v285: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:30:54.989 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:30:54 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:30:54.441218+0000 mon.smithi040 (mon.0) 255 : audit [DBG] from='client.? 172.21.15.66:0/2168050709' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-11-17T05:30:55.889 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2023-11-17T05:30:55.889 DEBUG:teuthology.orchestra.run.smithi066:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.4 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ed6470e2-8508-11ee-8dbe-212e2dc638e7 -- ceph mon dump -f json 2023-11-17T05:30:56.992 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:30:56 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:30:55.253714+0000 mgr.smithi040.ogcprl (mgr.14164) 300 : cluster [DBG] pgmap v286: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:30:57.055 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:57.054+0000 7f2817021700 1 -- 172.21.15.66:0/2103099842 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f28100f62a0 msgr2=0x7f28100f66c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:30:57.056 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:57.054+0000 7f2817021700 1 --2- 172.21.15.66:0/2103099842 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f28100f62a0 0x7f28100f66c0 secure :-1 s=READY pgs=311 cs=0 l=1 rev1=1 rx=0x7f280c009fb0 tx=0x7f280c009b30).stop 2023-11-17T05:30:57.056 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:57.054+0000 7f2817021700 1 -- 172.21.15.66:0/2103099842 shutdown_connections 2023-11-17T05:30:57.056 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:57.054+0000 7f2817021700 1 --2- 172.21.15.66:0/2103099842 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f28100f62a0 0x7f28100f66c0 unknown :-1 s=CLOSED pgs=311 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:30:57.056 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:57.054+0000 7f2817021700 1 -- 172.21.15.66:0/2103099842 >> 172.21.15.66:0/2103099842 conn(0x7f28100f1aa0 msgr2=0x7f28100f3ec0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:30:57.056 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:57.054+0000 7f2817021700 1 -- 172.21.15.66:0/2103099842 shutdown_connections 2023-11-17T05:30:57.056 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:57.054+0000 7f2817021700 1 -- 172.21.15.66:0/2103099842 wait complete. 2023-11-17T05:30:57.056 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:57.054+0000 7f2817021700 1 Processor -- start 2023-11-17T05:30:57.056 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:57.054+0000 7f2817021700 1 -- start start 2023-11-17T05:30:57.056 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:57.054+0000 7f2817021700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f28100f62a0 0x7f28101064b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:30:57.056 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:57.054+0000 7f2817021700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f28101069a0 con 0x7f28100f62a0 2023-11-17T05:30:57.058 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:57.055+0000 7f281601f700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f28100f62a0 0x7f28101064b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:30:57.058 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:57.055+0000 7f281601f700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f28100f62a0 0x7f28101064b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.66:37220/0 (socket says 172.21.15.66:37220) 2023-11-17T05:30:57.058 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:57.055+0000 7f281601f700 1 -- 172.21.15.66:0/2601725466 learned_addr learned my addr 172.21.15.66:0/2601725466 (peer_addr_for_me v2:172.21.15.66:0/0) 2023-11-17T05:30:57.058 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:57.055+0000 7f281601f700 1 -- 172.21.15.66:0/2601725466 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f280c005040 con 0x7f28100f62a0 2023-11-17T05:30:57.058 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:57.055+0000 7f281601f700 1 --2- 172.21.15.66:0/2601725466 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f28100f62a0 0x7f28101064b0 secure :-1 s=READY pgs=312 cs=0 l=1 rev1=1 rx=0x7f280c009830 tx=0x7f280c00b040).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:30:57.058 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:57.055+0000 7f2802ffd700 1 -- 172.21.15.66:0/2601725466 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f280c009320 con 0x7f28100f62a0 2023-11-17T05:30:57.058 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:57.055+0000 7f2817021700 1 -- 172.21.15.66:0/2601725466 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f2810106ba0 con 0x7f28100f62a0 2023-11-17T05:30:57.058 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:57.055+0000 7f2802ffd700 1 -- 172.21.15.66:0/2601725466 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(25 keys) v1 ==== 1074+0+0 (secure 0 0 0) 0x7f280c0136c0 con 0x7f28100f62a0 2023-11-17T05:30:57.059 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:57.055+0000 7f2802ffd700 1 -- 172.21.15.66:0/2601725466 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f280c01ee90 con 0x7f28100f62a0 2023-11-17T05:30:57.059 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:57.055+0000 7f2817021700 1 -- 172.21.15.66:0/2601725466 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f2810107030 con 0x7f28100f62a0 2023-11-17T05:30:57.059 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:57.056+0000 7f2802ffd700 1 -- 172.21.15.66:0/2601725466 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 17) v1 ==== 36269+0+0 (secure 0 0 0) 0x7f280c01e420 con 0x7f28100f62a0 2023-11-17T05:30:57.059 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:57.056+0000 7f2817021700 1 -- 172.21.15.66:0/2601725466 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f2810045ad0 con 0x7f28100f62a0 2023-11-17T05:30:57.059 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:57.056+0000 7f2802ffd700 1 --2- 172.21.15.66:0/2601725466 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f27fc02ff10 0x7f27fc0323d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:30:57.059 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:57.056+0000 7f2802ffd700 1 -- 172.21.15.66:0/2601725466 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(5..5 src has 1..5) v4 ==== 1270+0+0 (secure 0 0 0) 0x7f280c0410b0 con 0x7f28100f62a0 2023-11-17T05:30:57.059 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:57.057+0000 7f281581e700 1 --2- 172.21.15.66:0/2601725466 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f27fc02ff10 0x7f27fc0323d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:30:57.059 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:57.057+0000 7f281581e700 1 --2- 172.21.15.66:0/2601725466 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f27fc02ff10 0x7f27fc0323d0 secure :-1 s=READY pgs=121 cs=0 l=1 rev1=1 rx=0x7f28040096a0 tx=0x7f2804006b40).ready entity=mgr.14164 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:30:57.062 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:57.061+0000 7f2802ffd700 1 -- 172.21.15.66:0/2601725466 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+152212 (secure 0 0 0) 0x7f280c0095f0 con 0x7f28100f62a0 2023-11-17T05:30:57.282 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:57.281+0000 7f2817021700 1 -- 172.21.15.66:0/2601725466 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "mon dump", "format": "json"} v 0) v1 -- 0x7f2810107ee0 con 0x7f28100f62a0 2023-11-17T05:30:57.282 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:57.281+0000 7f2802ffd700 1 -- 172.21.15.66:0/2601725466 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mon_command_ack([{"prefix": "mon dump", "format": "json"}]=0 dumped monmap epoch 1 v1) v1 ==== 95+0+690 (secure 0 0 0) 0x7f280c02c2d0 con 0x7f28100f62a0 2023-11-17T05:30:57.283 INFO:teuthology.orchestra.run.smithi066.stdout: 2023-11-17T05:30:57.283 INFO:teuthology.orchestra.run.smithi066.stdout:{"epoch":1,"fsid":"ed6470e2-8508-11ee-8dbe-212e2dc638e7","modified":"2023-11-17T05:20:13.934038Z","created":"2023-11-17T05:20:13.934038Z","min_mon_release":16,"min_mon_release_name":"pacific","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging"],"optional":[]},"mons":[{"rank":0,"name":"smithi040","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:3300","nonce":0},{"type":"v1","addr":"172.21.15.40:6789","nonce":0}]},"addr":"172.21.15.40:6789/0","public_addr":"172.21.15.40:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2023-11-17T05:30:57.285 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:57.283+0000 7f2817021700 1 -- 172.21.15.66:0/2601725466 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f27fc02ff10 msgr2=0x7f27fc0323d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:30:57.285 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:57.283+0000 7f2817021700 1 --2- 172.21.15.66:0/2601725466 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f27fc02ff10 0x7f27fc0323d0 secure :-1 s=READY pgs=121 cs=0 l=1 rev1=1 rx=0x7f28040096a0 tx=0x7f2804006b40).stop 2023-11-17T05:30:57.285 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:57.283+0000 7f2817021700 1 -- 172.21.15.66:0/2601725466 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f28100f62a0 msgr2=0x7f28101064b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:30:57.285 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:57.283+0000 7f2817021700 1 --2- 172.21.15.66:0/2601725466 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f28100f62a0 0x7f28101064b0 secure :-1 s=READY pgs=312 cs=0 l=1 rev1=1 rx=0x7f280c009830 tx=0x7f280c00b040).stop 2023-11-17T05:30:57.285 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:57.283+0000 7f2817021700 1 -- 172.21.15.66:0/2601725466 shutdown_connections 2023-11-17T05:30:57.285 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:57.283+0000 7f2817021700 1 --2- 172.21.15.66:0/2601725466 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f27fc02ff10 0x7f27fc0323d0 unknown :-1 s=CLOSED pgs=121 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:30:57.285 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:57.283+0000 7f2817021700 1 --2- 172.21.15.66:0/2601725466 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f28100f62a0 0x7f28101064b0 unknown :-1 s=CLOSED pgs=312 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:30:57.286 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:57.283+0000 7f2817021700 1 -- 172.21.15.66:0/2601725466 >> 172.21.15.66:0/2601725466 conn(0x7f28100f1aa0 msgr2=0x7f28100f2510 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:30:57.286 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:57.283+0000 7f2817021700 1 -- 172.21.15.66:0/2601725466 shutdown_connections 2023-11-17T05:30:57.286 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:57.283+0000 7f2817021700 1 -- 172.21.15.66:0/2601725466 wait complete. 2023-11-17T05:30:57.286 INFO:teuthology.orchestra.run.smithi066.stderr:dumped monmap epoch 1 2023-11-17T05:30:57.989 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:30:57 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:30:57.282998+0000 mon.smithi040 (mon.0) 256 : audit [DBG] from='client.? 172.21.15.66:0/2601725466' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-11-17T05:30:58.714 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2023-11-17T05:30:58.714 DEBUG:teuthology.orchestra.run.smithi066:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.4 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ed6470e2-8508-11ee-8dbe-212e2dc638e7 -- ceph mon dump -f json 2023-11-17T05:30:58.988 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:30:58 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:30:57.254197+0000 mgr.smithi040.ogcprl (mgr.14164) 301 : cluster [DBG] pgmap v287: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:30:59.891 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:59.889+0000 7f7222cdb700 1 -- 172.21.15.66:0/1880938594 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f721c060280 msgr2=0x7f721c0f90f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:30:59.891 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:59.889+0000 7f7222cdb700 1 --2- 172.21.15.66:0/1880938594 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f721c060280 0x7f721c0f90f0 secure :-1 s=READY pgs=313 cs=0 l=1 rev1=1 rx=0x7f7210009fb0 tx=0x7f7210009b30).stop 2023-11-17T05:30:59.891 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:59.889+0000 7f7222cdb700 1 -- 172.21.15.66:0/1880938594 shutdown_connections 2023-11-17T05:30:59.891 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:59.889+0000 7f7222cdb700 1 --2- 172.21.15.66:0/1880938594 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f721c060280 0x7f721c0f90f0 unknown :-1 s=CLOSED pgs=313 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:30:59.891 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:59.889+0000 7f7222cdb700 1 -- 172.21.15.66:0/1880938594 >> 172.21.15.66:0/1880938594 conn(0x7f721c0f0b70 msgr2=0x7f721c0f2f90 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:30:59.891 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:59.889+0000 7f7222cdb700 1 -- 172.21.15.66:0/1880938594 shutdown_connections 2023-11-17T05:30:59.891 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:59.889+0000 7f7222cdb700 1 -- 172.21.15.66:0/1880938594 wait complete. 2023-11-17T05:30:59.891 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:59.890+0000 7f7222cdb700 1 Processor -- start 2023-11-17T05:30:59.892 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:59.890+0000 7f7222cdb700 1 -- start start 2023-11-17T05:30:59.893 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:59.890+0000 7f7222cdb700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f721c060280 0x7f721c0f6b70 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:30:59.893 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:59.890+0000 7f7222cdb700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f721c0f4ff0 con 0x7f721c060280 2023-11-17T05:30:59.893 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:59.890+0000 7f7220a77700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f721c060280 0x7f721c0f6b70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:30:59.893 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:59.890+0000 7f7220a77700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f721c060280 0x7f721c0f6b70 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.66:37232/0 (socket says 172.21.15.66:37232) 2023-11-17T05:30:59.893 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:59.890+0000 7f7220a77700 1 -- 172.21.15.66:0/3945425995 learned_addr learned my addr 172.21.15.66:0/3945425995 (peer_addr_for_me v2:172.21.15.66:0/0) 2023-11-17T05:30:59.893 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:59.891+0000 7f7220a77700 1 -- 172.21.15.66:0/3945425995 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f7210005040 con 0x7f721c060280 2023-11-17T05:30:59.893 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:59.891+0000 7f7220a77700 1 --2- 172.21.15.66:0/3945425995 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f721c060280 0x7f721c0f6b70 secure :-1 s=READY pgs=314 cs=0 l=1 rev1=1 rx=0x7f7210013510 tx=0x7f721000b040).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:30:59.893 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:59.891+0000 7f72197fa700 1 -- 172.21.15.66:0/3945425995 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f7210009320 con 0x7f721c060280 2023-11-17T05:30:59.893 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:59.891+0000 7f72197fa700 1 -- 172.21.15.66:0/3945425995 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(25 keys) v1 ==== 1074+0+0 (secure 0 0 0) 0x7f7210013f00 con 0x7f721c060280 2023-11-17T05:30:59.893 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:59.891+0000 7f7222cdb700 1 -- 172.21.15.66:0/3945425995 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f721c0f5190 con 0x7f721c060280 2023-11-17T05:30:59.893 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:59.891+0000 7f72197fa700 1 -- 172.21.15.66:0/3945425995 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f7210014dc0 con 0x7f721c060280 2023-11-17T05:30:59.894 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:59.891+0000 7f7222cdb700 1 -- 172.21.15.66:0/3945425995 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f721c0f5620 con 0x7f721c060280 2023-11-17T05:30:59.894 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:59.892+0000 7f72197fa700 1 -- 172.21.15.66:0/3945425995 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 17) v1 ==== 36269+0+0 (secure 0 0 0) 0x7f721001d3d0 con 0x7f721c060280 2023-11-17T05:30:59.894 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:59.892+0000 7f7222cdb700 1 -- 172.21.15.66:0/3945425995 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f721c059100 con 0x7f721c060280 2023-11-17T05:30:59.895 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:59.892+0000 7f72197fa700 1 --2- 172.21.15.66:0/3945425995 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f720c02ff60 0x7f720c032420 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:30:59.895 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:59.892+0000 7f72197fa700 1 -- 172.21.15.66:0/3945425995 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(5..5 src has 1..5) v4 ==== 1270+0+0 (secure 0 0 0) 0x7f72100413a0 con 0x7f721c060280 2023-11-17T05:30:59.895 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:59.892+0000 7f721bfff700 1 --2- 172.21.15.66:0/3945425995 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f720c02ff60 0x7f720c032420 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:30:59.895 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:59.893+0000 7f721bfff700 1 --2- 172.21.15.66:0/3945425995 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f720c02ff60 0x7f720c032420 secure :-1 s=READY pgs=122 cs=0 l=1 rev1=1 rx=0x7f72080096a0 tx=0x7f7208006b40).ready entity=mgr.14164 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:30:59.898 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:30:59.897+0000 7f72197fa700 1 -- 172.21.15.66:0/3945425995 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+152212 (secure 0 0 0) 0x7f721001cb10 con 0x7f721c060280 2023-11-17T05:31:00.117 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:00.116+0000 7f7222cdb700 1 -- 172.21.15.66:0/3945425995 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "mon dump", "format": "json"} v 0) v1 -- 0x7f721c0fbf50 con 0x7f721c060280 2023-11-17T05:31:00.118 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:00.117+0000 7f72197fa700 1 -- 172.21.15.66:0/3945425995 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mon_command_ack([{"prefix": "mon dump", "format": "json"}]=0 dumped monmap epoch 1 v1) v1 ==== 95+0+690 (secure 0 0 0) 0x7f721001cb10 con 0x7f721c060280 2023-11-17T05:31:00.118 INFO:teuthology.orchestra.run.smithi066.stdout: 2023-11-17T05:31:00.118 INFO:teuthology.orchestra.run.smithi066.stdout:{"epoch":1,"fsid":"ed6470e2-8508-11ee-8dbe-212e2dc638e7","modified":"2023-11-17T05:20:13.934038Z","created":"2023-11-17T05:20:13.934038Z","min_mon_release":16,"min_mon_release_name":"pacific","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging"],"optional":[]},"mons":[{"rank":0,"name":"smithi040","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:3300","nonce":0},{"type":"v1","addr":"172.21.15.40:6789","nonce":0}]},"addr":"172.21.15.40:6789/0","public_addr":"172.21.15.40:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2023-11-17T05:31:00.120 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:00.118+0000 7f7222cdb700 1 -- 172.21.15.66:0/3945425995 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f720c02ff60 msgr2=0x7f720c032420 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:31:00.120 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:00.118+0000 7f7222cdb700 1 --2- 172.21.15.66:0/3945425995 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f720c02ff60 0x7f720c032420 secure :-1 s=READY pgs=122 cs=0 l=1 rev1=1 rx=0x7f72080096a0 tx=0x7f7208006b40).stop 2023-11-17T05:31:00.120 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:00.118+0000 7f7222cdb700 1 -- 172.21.15.66:0/3945425995 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f721c060280 msgr2=0x7f721c0f6b70 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:31:00.120 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:00.118+0000 7f7222cdb700 1 --2- 172.21.15.66:0/3945425995 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f721c060280 0x7f721c0f6b70 secure :-1 s=READY pgs=314 cs=0 l=1 rev1=1 rx=0x7f7210013510 tx=0x7f721000b040).stop 2023-11-17T05:31:00.120 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:00.119+0000 7f7222cdb700 1 -- 172.21.15.66:0/3945425995 shutdown_connections 2023-11-17T05:31:00.120 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:00.119+0000 7f7222cdb700 1 --2- 172.21.15.66:0/3945425995 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f720c02ff60 0x7f720c032420 unknown :-1 s=CLOSED pgs=122 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:31:00.120 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:00.119+0000 7f7222cdb700 1 --2- 172.21.15.66:0/3945425995 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f721c060280 0x7f721c0f6b70 unknown :-1 s=CLOSED pgs=314 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:31:00.121 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:00.119+0000 7f7222cdb700 1 -- 172.21.15.66:0/3945425995 >> 172.21.15.66:0/3945425995 conn(0x7f721c0f0b70 msgr2=0x7f721c0f1660 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:31:00.121 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:00.119+0000 7f7222cdb700 1 -- 172.21.15.66:0/3945425995 shutdown_connections 2023-11-17T05:31:00.121 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:00.119+0000 7f7222cdb700 1 -- 172.21.15.66:0/3945425995 wait complete. 2023-11-17T05:31:00.121 INFO:teuthology.orchestra.run.smithi066.stderr:dumped monmap epoch 1 2023-11-17T05:31:00.988 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:31:00 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:30:59.254593+0000 mgr.smithi040.ogcprl (mgr.14164) 302 : cluster [DBG] pgmap v288: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:31:00.989 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:31:00 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:31:00.118568+0000 mon.smithi040 (mon.0) 257 : audit [DBG] from='client.? 172.21.15.66:0/3945425995' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-11-17T05:31:01.518 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2023-11-17T05:31:01.518 DEBUG:teuthology.orchestra.run.smithi066:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.4 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ed6470e2-8508-11ee-8dbe-212e2dc638e7 -- ceph mon dump -f json 2023-11-17T05:31:02.682 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:02.678+0000 7fde34fe5700 1 -- 172.21.15.66:0/3931132707 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fde300f6510 msgr2=0x7fde300f6930 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:31:02.683 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:02.678+0000 7fde34fe5700 1 --2- 172.21.15.66:0/3931132707 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fde300f6510 0x7fde300f6930 secure :-1 s=READY pgs=315 cs=0 l=1 rev1=1 rx=0x7fde20009fb0 tx=0x7fde20009b30).stop 2023-11-17T05:31:02.683 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:02.678+0000 7fde34fe5700 1 -- 172.21.15.66:0/3931132707 shutdown_connections 2023-11-17T05:31:02.683 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:02.678+0000 7fde34fe5700 1 --2- 172.21.15.66:0/3931132707 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fde300f6510 0x7fde300f6930 unknown :-1 s=CLOSED pgs=315 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:31:02.683 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:02.678+0000 7fde34fe5700 1 -- 172.21.15.66:0/3931132707 >> 172.21.15.66:0/3931132707 conn(0x7fde300f1c70 msgr2=0x7fde300f4090 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:31:02.683 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:02.678+0000 7fde34fe5700 1 -- 172.21.15.66:0/3931132707 shutdown_connections 2023-11-17T05:31:02.683 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:02.678+0000 7fde34fe5700 1 -- 172.21.15.66:0/3931132707 wait complete. 2023-11-17T05:31:02.683 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:02.679+0000 7fde34fe5700 1 Processor -- start 2023-11-17T05:31:02.683 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:02.679+0000 7fde34fe5700 1 -- start start 2023-11-17T05:31:02.683 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:02.679+0000 7fde34fe5700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fde300f6510 0x7fde30106730 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:31:02.683 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:02.679+0000 7fde34fe5700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fde30106c20 con 0x7fde300f6510 2023-11-17T05:31:02.683 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:02.679+0000 7fde2e59c700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fde300f6510 0x7fde30106730 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:31:02.683 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:02.679+0000 7fde2e59c700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fde300f6510 0x7fde30106730 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.66:50928/0 (socket says 172.21.15.66:50928) 2023-11-17T05:31:02.684 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:02.679+0000 7fde2e59c700 1 -- 172.21.15.66:0/328473753 learned_addr learned my addr 172.21.15.66:0/328473753 (peer_addr_for_me v2:172.21.15.66:0/0) 2023-11-17T05:31:02.684 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:02.679+0000 7fde2e59c700 1 -- 172.21.15.66:0/328473753 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fde20005040 con 0x7fde300f6510 2023-11-17T05:31:02.684 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:02.679+0000 7fde2e59c700 1 --2- 172.21.15.66:0/328473753 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fde300f6510 0x7fde30106730 secure :-1 s=READY pgs=316 cs=0 l=1 rev1=1 rx=0x7fde20013510 tx=0x7fde2000b040).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:31:02.684 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:02.680+0000 7fde1effd700 1 -- 172.21.15.66:0/328473753 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7fde20009320 con 0x7fde300f6510 2023-11-17T05:31:02.684 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:02.680+0000 7fde34fe5700 1 -- 172.21.15.66:0/328473753 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fde30106e20 con 0x7fde300f6510 2023-11-17T05:31:02.684 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:02.680+0000 7fde1effd700 1 -- 172.21.15.66:0/328473753 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(25 keys) v1 ==== 1074+0+0 (secure 0 0 0) 0x7fde20013f00 con 0x7fde300f6510 2023-11-17T05:31:02.684 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:02.680+0000 7fde1effd700 1 -- 172.21.15.66:0/328473753 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7fde20014d90 con 0x7fde300f6510 2023-11-17T05:31:02.684 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:02.680+0000 7fde34fe5700 1 -- 172.21.15.66:0/328473753 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fde301072b0 con 0x7fde300f6510 2023-11-17T05:31:02.684 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:02.680+0000 7fde1effd700 1 -- 172.21.15.66:0/328473753 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 17) v1 ==== 36269+0+0 (secure 0 0 0) 0x7fde200095f0 con 0x7fde300f6510 2023-11-17T05:31:02.684 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:02.680+0000 7fde34fe5700 1 -- 172.21.15.66:0/328473753 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fde30045ad0 con 0x7fde300f6510 2023-11-17T05:31:02.684 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:02.681+0000 7fde1effd700 1 --2- 172.21.15.66:0/328473753 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fde1802ff60 0x7fde18032420 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:31:02.684 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:02.681+0000 7fde1effd700 1 -- 172.21.15.66:0/328473753 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(5..5 src has 1..5) v4 ==== 1270+0+0 (secure 0 0 0) 0x7fde20041000 con 0x7fde300f6510 2023-11-17T05:31:02.684 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:02.681+0000 7fde2dd9b700 1 --2- 172.21.15.66:0/328473753 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fde1802ff60 0x7fde18032420 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:31:02.685 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:02.681+0000 7fde2dd9b700 1 --2- 172.21.15.66:0/328473753 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fde1802ff60 0x7fde18032420 secure :-1 s=READY pgs=123 cs=0 l=1 rev1=1 rx=0x7fde240096a0 tx=0x7fde24006b40).ready entity=mgr.14164 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:31:02.687 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:02.685+0000 7fde1effd700 1 -- 172.21.15.66:0/328473753 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+152212 (secure 0 0 0) 0x7fde20014420 con 0x7fde300f6510 2023-11-17T05:31:02.906 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:02.905+0000 7fde34fe5700 1 -- 172.21.15.66:0/328473753 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "mon dump", "format": "json"} v 0) v1 -- 0x7fde300fbf60 con 0x7fde300f6510 2023-11-17T05:31:02.906 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:02.905+0000 7fde1effd700 1 -- 172.21.15.66:0/328473753 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mon_command_ack([{"prefix": "mon dump", "format": "json"}]=0 dumped monmap epoch 1 v1) v1 ==== 95+0+690 (secure 0 0 0) 0x7fde2001d7f0 con 0x7fde300f6510 2023-11-17T05:31:02.907 INFO:teuthology.orchestra.run.smithi066.stdout: 2023-11-17T05:31:02.907 INFO:teuthology.orchestra.run.smithi066.stdout:{"epoch":1,"fsid":"ed6470e2-8508-11ee-8dbe-212e2dc638e7","modified":"2023-11-17T05:20:13.934038Z","created":"2023-11-17T05:20:13.934038Z","min_mon_release":16,"min_mon_release_name":"pacific","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging"],"optional":[]},"mons":[{"rank":0,"name":"smithi040","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:3300","nonce":0},{"type":"v1","addr":"172.21.15.40:6789","nonce":0}]},"addr":"172.21.15.40:6789/0","public_addr":"172.21.15.40:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2023-11-17T05:31:02.908 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:02.907+0000 7fde34fe5700 1 -- 172.21.15.66:0/328473753 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fde1802ff60 msgr2=0x7fde18032420 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:31:02.908 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:02.907+0000 7fde34fe5700 1 --2- 172.21.15.66:0/328473753 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fde1802ff60 0x7fde18032420 secure :-1 s=READY pgs=123 cs=0 l=1 rev1=1 rx=0x7fde240096a0 tx=0x7fde24006b40).stop 2023-11-17T05:31:02.909 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:02.907+0000 7fde34fe5700 1 -- 172.21.15.66:0/328473753 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fde300f6510 msgr2=0x7fde30106730 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:31:02.909 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:02.907+0000 7fde34fe5700 1 --2- 172.21.15.66:0/328473753 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fde300f6510 0x7fde30106730 secure :-1 s=READY pgs=316 cs=0 l=1 rev1=1 rx=0x7fde20013510 tx=0x7fde2000b040).stop 2023-11-17T05:31:02.909 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:02.907+0000 7fde34fe5700 1 -- 172.21.15.66:0/328473753 shutdown_connections 2023-11-17T05:31:02.909 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:02.907+0000 7fde34fe5700 1 --2- 172.21.15.66:0/328473753 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fde1802ff60 0x7fde18032420 unknown :-1 s=CLOSED pgs=123 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:31:02.909 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:02.907+0000 7fde34fe5700 1 --2- 172.21.15.66:0/328473753 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fde300f6510 0x7fde30106730 unknown :-1 s=CLOSED pgs=316 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:31:02.909 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:02.907+0000 7fde34fe5700 1 -- 172.21.15.66:0/328473753 >> 172.21.15.66:0/328473753 conn(0x7fde300f1c70 msgr2=0x7fde300f2760 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:31:02.909 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:02.907+0000 7fde34fe5700 1 -- 172.21.15.66:0/328473753 shutdown_connections 2023-11-17T05:31:02.909 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:02.907+0000 7fde34fe5700 1 -- 172.21.15.66:0/328473753 wait complete. 2023-11-17T05:31:02.910 INFO:teuthology.orchestra.run.smithi066.stderr:dumped monmap epoch 1 2023-11-17T05:31:02.989 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:31:02 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:31:01.254910+0000 mgr.smithi040.ogcprl (mgr.14164) 303 : cluster [DBG] pgmap v289: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:31:03.989 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:31:03 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:31:02.907022+0000 mon.smithi040 (mon.0) 258 : audit [DBG] from='client.? 172.21.15.66:0/328473753' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-11-17T05:31:04.302 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2023-11-17T05:31:04.303 DEBUG:teuthology.orchestra.run.smithi066:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.4 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ed6470e2-8508-11ee-8dbe-212e2dc638e7 -- ceph mon dump -f json 2023-11-17T05:31:04.988 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:31:04 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:31:03.255190+0000 mgr.smithi040.ogcprl (mgr.14164) 304 : cluster [DBG] pgmap v290: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:31:05.555 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:05.553+0000 7fac28a36700 1 -- 172.21.15.66:0/3667066414 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fac240f51a0 msgr2=0x7fac240f55c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:31:05.555 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:05.553+0000 7fac28a36700 1 --2- 172.21.15.66:0/3667066414 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fac240f51a0 0x7fac240f55c0 secure :-1 s=READY pgs=317 cs=0 l=1 rev1=1 rx=0x7fac14009fb0 tx=0x7fac14009b30).stop 2023-11-17T05:31:05.555 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:05.553+0000 7fac28a36700 1 -- 172.21.15.66:0/3667066414 shutdown_connections 2023-11-17T05:31:05.555 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:05.553+0000 7fac28a36700 1 --2- 172.21.15.66:0/3667066414 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fac240f51a0 0x7fac240f55c0 unknown :-1 s=CLOSED pgs=317 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:31:05.556 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:05.553+0000 7fac28a36700 1 -- 172.21.15.66:0/3667066414 >> 172.21.15.66:0/3667066414 conn(0x7fac240f09a0 msgr2=0x7fac240f2dc0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:31:05.556 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:05.553+0000 7fac28a36700 1 -- 172.21.15.66:0/3667066414 shutdown_connections 2023-11-17T05:31:05.556 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:05.553+0000 7fac28a36700 1 -- 172.21.15.66:0/3667066414 wait complete. 2023-11-17T05:31:05.556 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:05.553+0000 7fac28a36700 1 Processor -- start 2023-11-17T05:31:05.556 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:05.553+0000 7fac28a36700 1 -- start start 2023-11-17T05:31:05.556 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:05.554+0000 7fac28a36700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fac240f51a0 0x7fac241064e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:31:05.556 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:05.554+0000 7fac28a36700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fac241069d0 con 0x7fac240f51a0 2023-11-17T05:31:05.557 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:05.554+0000 7fac237fe700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fac240f51a0 0x7fac241064e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:31:05.558 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:05.554+0000 7fac237fe700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fac240f51a0 0x7fac241064e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.66:50942/0 (socket says 172.21.15.66:50942) 2023-11-17T05:31:05.558 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:05.554+0000 7fac237fe700 1 -- 172.21.15.66:0/1562129381 learned_addr learned my addr 172.21.15.66:0/1562129381 (peer_addr_for_me v2:172.21.15.66:0/0) 2023-11-17T05:31:05.558 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:05.554+0000 7fac237fe700 1 -- 172.21.15.66:0/1562129381 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fac14005040 con 0x7fac240f51a0 2023-11-17T05:31:05.558 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:05.554+0000 7fac237fe700 1 --2- 172.21.15.66:0/1562129381 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fac240f51a0 0x7fac241064e0 secure :-1 s=READY pgs=318 cs=0 l=1 rev1=1 rx=0x7fac14009830 tx=0x7fac1400b040).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:31:05.558 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:05.554+0000 7fac1bfff700 1 -- 172.21.15.66:0/1562129381 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7fac14009320 con 0x7fac240f51a0 2023-11-17T05:31:05.558 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:05.554+0000 7fac28a36700 1 -- 172.21.15.66:0/1562129381 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fac24106bd0 con 0x7fac240f51a0 2023-11-17T05:31:05.558 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:05.554+0000 7fac1bfff700 1 -- 172.21.15.66:0/1562129381 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(25 keys) v1 ==== 1074+0+0 (secure 0 0 0) 0x7fac140136c0 con 0x7fac240f51a0 2023-11-17T05:31:05.558 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:05.554+0000 7fac1bfff700 1 -- 172.21.15.66:0/1562129381 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7fac1401ee90 con 0x7fac240f51a0 2023-11-17T05:31:05.558 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:05.554+0000 7fac28a36700 1 -- 172.21.15.66:0/1562129381 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fac24107060 con 0x7fac240f51a0 2023-11-17T05:31:05.558 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:05.555+0000 7fac1bfff700 1 -- 172.21.15.66:0/1562129381 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 17) v1 ==== 36269+0+0 (secure 0 0 0) 0x7fac1401e420 con 0x7fac240f51a0 2023-11-17T05:31:05.558 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:05.555+0000 7fac28a36700 1 -- 172.21.15.66:0/1562129381 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fac24045ad0 con 0x7fac240f51a0 2023-11-17T05:31:05.558 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:05.555+0000 7fac1bfff700 1 --2- 172.21.15.66:0/1562129381 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fac1002ff10 0x7fac100323d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:31:05.558 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:05.555+0000 7fac1bfff700 1 -- 172.21.15.66:0/1562129381 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(5..5 src has 1..5) v4 ==== 1270+0+0 (secure 0 0 0) 0x7fac14041120 con 0x7fac240f51a0 2023-11-17T05:31:05.559 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:05.556+0000 7fac22ffd700 1 --2- 172.21.15.66:0/1562129381 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fac1002ff10 0x7fac100323d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:31:05.559 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:05.556+0000 7fac22ffd700 1 --2- 172.21.15.66:0/1562129381 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fac1002ff10 0x7fac100323d0 secure :-1 s=READY pgs=124 cs=0 l=1 rev1=1 rx=0x7fac0c0096a0 tx=0x7fac0c006b40).ready entity=mgr.14164 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:31:05.561 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:05.560+0000 7fac1bfff700 1 -- 172.21.15.66:0/1562129381 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+152212 (secure 0 0 0) 0x7fac140095f0 con 0x7fac240f51a0 2023-11-17T05:31:05.778 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:05.777+0000 7fac28a36700 1 -- 172.21.15.66:0/1562129381 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "mon dump", "format": "json"} v 0) v1 -- 0x7fac24107f10 con 0x7fac240f51a0 2023-11-17T05:31:05.779 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:05.777+0000 7fac1bfff700 1 -- 172.21.15.66:0/1562129381 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mon_command_ack([{"prefix": "mon dump", "format": "json"}]=0 dumped monmap epoch 1 v1) v1 ==== 95+0+690 (secure 0 0 0) 0x7fac1401cb10 con 0x7fac240f51a0 2023-11-17T05:31:05.779 INFO:teuthology.orchestra.run.smithi066.stdout: 2023-11-17T05:31:05.779 INFO:teuthology.orchestra.run.smithi066.stdout:{"epoch":1,"fsid":"ed6470e2-8508-11ee-8dbe-212e2dc638e7","modified":"2023-11-17T05:20:13.934038Z","created":"2023-11-17T05:20:13.934038Z","min_mon_release":16,"min_mon_release_name":"pacific","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging"],"optional":[]},"mons":[{"rank":0,"name":"smithi040","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:3300","nonce":0},{"type":"v1","addr":"172.21.15.40:6789","nonce":0}]},"addr":"172.21.15.40:6789/0","public_addr":"172.21.15.40:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2023-11-17T05:31:05.780 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:05.779+0000 7fac28a36700 1 -- 172.21.15.66:0/1562129381 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fac1002ff10 msgr2=0x7fac100323d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:31:05.781 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:05.779+0000 7fac28a36700 1 --2- 172.21.15.66:0/1562129381 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fac1002ff10 0x7fac100323d0 secure :-1 s=READY pgs=124 cs=0 l=1 rev1=1 rx=0x7fac0c0096a0 tx=0x7fac0c006b40).stop 2023-11-17T05:31:05.781 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:05.779+0000 7fac28a36700 1 -- 172.21.15.66:0/1562129381 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fac240f51a0 msgr2=0x7fac241064e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:31:05.781 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:05.779+0000 7fac28a36700 1 --2- 172.21.15.66:0/1562129381 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fac240f51a0 0x7fac241064e0 secure :-1 s=READY pgs=318 cs=0 l=1 rev1=1 rx=0x7fac14009830 tx=0x7fac1400b040).stop 2023-11-17T05:31:05.781 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:05.779+0000 7fac28a36700 1 -- 172.21.15.66:0/1562129381 shutdown_connections 2023-11-17T05:31:05.781 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:05.779+0000 7fac28a36700 1 --2- 172.21.15.66:0/1562129381 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fac1002ff10 0x7fac100323d0 unknown :-1 s=CLOSED pgs=124 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:31:05.781 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:05.779+0000 7fac28a36700 1 --2- 172.21.15.66:0/1562129381 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fac240f51a0 0x7fac241064e0 unknown :-1 s=CLOSED pgs=318 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:31:05.781 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:05.779+0000 7fac28a36700 1 -- 172.21.15.66:0/1562129381 >> 172.21.15.66:0/1562129381 conn(0x7fac240f09a0 msgr2=0x7fac2405fe20 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:31:05.781 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:05.779+0000 7fac28a36700 1 -- 172.21.15.66:0/1562129381 shutdown_connections 2023-11-17T05:31:05.781 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:05.779+0000 7fac28a36700 1 -- 172.21.15.66:0/1562129381 wait complete. 2023-11-17T05:31:05.782 INFO:teuthology.orchestra.run.smithi066.stderr:dumped monmap epoch 1 2023-11-17T05:31:05.988 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:31:05 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:31:05.255437+0000 mgr.smithi040.ogcprl (mgr.14164) 305 : cluster [DBG] pgmap v291: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:31:06.989 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:31:06 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:31:05.779034+0000 mon.smithi040 (mon.0) 259 : audit [DBG] from='client.? 172.21.15.66:0/1562129381' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-11-17T05:31:07.150 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2023-11-17T05:31:07.150 DEBUG:teuthology.orchestra.run.smithi066:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.4 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ed6470e2-8508-11ee-8dbe-212e2dc638e7 -- ceph mon dump -f json 2023-11-17T05:31:07.828 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:31:07 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cephadm 2023-11-17T05:31:07.250973+0000 mgr.smithi040.ogcprl (mgr.14164) 306 : cephadm [ERR] cephadm exited with an error code: 1, stderr:Deploy daemon prometheus.smithi040 ... 2023-11-17T05:31:07.829 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:31:07 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: Non-zero exit code 125 from /bin/podman run --rm --ipc=host --no-hosts --net=host --entrypoint stat --init -e CONTAINER_IMAGE=docker.io/prom/prometheus:v2.18.1 -e NODE_NAME=smithi040 -e CEPH_USE_RANDOM_NONCE=1 docker.io/prom/prometheus:v2.18.1 -c %u %g /etc/prometheus 2023-11-17T05:31:07.829 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:31:07 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: stat: stderr Trying to pull docker.io/prom/prometheus:v2.18.1... 2023-11-17T05:31:07.829 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:31:07 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: stat: stderr time="2023-11-17T05:29:15Z" level=warning msg="Failed, retrying in 1s ... (1/3). Error: initializing source docker://prom/prometheus:v2.18.1: (Mirrors also failed: [docker-mirror.front.sepia.ceph.com:5000/prom/prometheus:v2.18.1: pinging container registry docker-mirror.front.sepia.ceph.com:5000: Get \"https://docker-mirror.front.sepia.ceph.com:5000/v2/\": dial tcp 172.21.0.79:5000: connect: no route to host]): docker.io/prom/prometheus:v2.18.1: reading manifest v2.18.1 in docker.io/prom/prometheus: toomanyrequests: You have reached your pull rate limit. You may increase the limit by authenticating and upgrading: https://www.docker.com/increase-rate-limit" 2023-11-17T05:31:07.829 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:31:07 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: stat: stderr time="2023-11-17T05:29:52Z" level=warning msg="Failed, retrying in 1s ... (2/3). Error: initializing source docker://prom/prometheus:v2.18.1: (Mirrors also failed: [docker-mirror.front.sepia.ceph.com:5000/prom/prometheus:v2.18.1: pinging container registry docker-mirror.front.sepia.ceph.com:5000: Get \"https://docker-mirror.front.sepia.ceph.com:5000/v2/\": dial tcp 172.21.0.79:5000: connect: no route to host]): docker.io/prom/prometheus:v2.18.1: reading manifest v2.18.1 in docker.io/prom/prometheus: toomanyrequests: You have reached your pull rate limit. You may increase the limit by authenticating and upgrading: https://www.docker.com/increase-rate-limit" 2023-11-17T05:31:07.829 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:31:07 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: stat: stderr time="2023-11-17T05:30:29Z" level=warning msg="Failed, retrying in 1s ... (3/3). Error: initializing source docker://prom/prometheus:v2.18.1: (Mirrors also failed: [docker-mirror.front.sepia.ceph.com:5000/prom/prometheus:v2.18.1: pinging container registry docker-mirror.front.sepia.ceph.com:5000: Get \"https://docker-mirror.front.sepia.ceph.com:5000/v2/\": dial tcp 172.21.0.79:5000: connect: no route to host]): docker.io/prom/prometheus:v2.18.1: reading manifest v2.18.1 in docker.io/prom/prometheus: toomanyrequests: You have reached your pull rate limit. You may increase the limit by authenticating and upgrading: https://www.docker.com/increase-rate-limit" 2023-11-17T05:31:07.829 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:31:07 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: stat: stderr Error: initializing source docker://prom/prometheus:v2.18.1: (Mirrors also failed: [docker-mirror.front.sepia.ceph.com:5000/prom/prometheus:v2.18.1: pinging container registry docker-mirror.front.sepia.ceph.com:5000: Get "https://docker-mirror.front.sepia.ceph.com:5000/v2/": dial tcp 172.21.0.79:5000: connect: no route to host]): docker.io/prom/prometheus:v2.18.1: reading manifest v2.18.1 in docker.io/prom/prometheus: toomanyrequests: You have reached your pull rate limit. You may increase the limit by authenticating and upgrading: https://www.docker.com/increase-rate-limit 2023-11-17T05:31:07.829 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:31:07 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: Traceback (most recent call last): 2023-11-17T05:31:07.829 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:31:07 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: File "/var/lib/ceph/ed6470e2-8508-11ee-8dbe-212e2dc638e7/cephadm.30cb78bdbbafb384af862e1c2292b944f15942b586128e91262b43e91e11ae90", line 8029, in 2023-11-17T05:31:07.829 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:31:07 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: main() 2023-11-17T05:31:07.830 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:31:07 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: File "/var/lib/ceph/ed6470e2-8508-11ee-8dbe-212e2dc638e7/cephadm.30cb78bdbbafb384af862e1c2292b944f15942b586128e91262b43e91e11ae90", line 8017, in main 2023-11-17T05:31:07.830 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:31:07 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: r = ctx.func(ctx) 2023-11-17T05:31:07.830 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:31:07 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: File "/var/lib/ceph/ed6470e2-8508-11ee-8dbe-212e2dc638e7/cephadm.30cb78bdbbafb384af862e1c2292b944f15942b586128e91262b43e91e11ae90", line 1760, in _default_image 2023-11-17T05:31:07.830 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:31:07 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: return func(ctx) 2023-11-17T05:31:07.830 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:31:07 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: File "/var/lib/ceph/ed6470e2-8508-11ee-8dbe-212e2dc638e7/cephadm.30cb78bdbbafb384af862e1c2292b944f15942b586128e91262b43e91e11ae90", line 4263, in command_deploy 2023-11-17T05:31:07.830 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:31:07 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: uid, gid = extract_uid_gid_monitoring(ctx, daemon_type) 2023-11-17T05:31:07.830 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:31:07 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: File "/var/lib/ceph/ed6470e2-8508-11ee-8dbe-212e2dc638e7/cephadm.30cb78bdbbafb384af862e1c2292b944f15942b586128e91262b43e91e11ae90", line 4185, in extract_uid_gid_monitoring 2023-11-17T05:31:07.830 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:31:07 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: uid, gid = extract_uid_gid(ctx, file_path='/etc/prometheus') 2023-11-17T05:31:07.830 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:31:07 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: File "/var/lib/ceph/ed6470e2-8508-11ee-8dbe-212e2dc638e7/cephadm.30cb78bdbbafb384af862e1c2292b944f15942b586128e91262b43e91e11ae90", line 2531, in extract_uid_gid 2023-11-17T05:31:07.830 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:31:07 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: raise RuntimeError('uid/gid not found') 2023-11-17T05:31:07.831 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:31:07 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: RuntimeError: uid/gid not found 2023-11-17T05:31:07.831 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:31:07 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: Traceback (most recent call last): 2023-11-17T05:31:07.831 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:31:07 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1144, in _remote_connection 2023-11-17T05:31:07.831 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:31:07 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: yield (conn, connr) 2023-11-17T05:31:07.831 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:31:07 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1059, in _run_cephadm 2023-11-17T05:31:07.831 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:31:07 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: code, '\n'.join(err))) 2023-11-17T05:31:07.831 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:31:07 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr:Deploy daemon prometheus.smithi040 ... 2023-11-17T05:31:07.831 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:31:07 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: Non-zero exit code 125 from /bin/podman run --rm --ipc=host --no-hosts --net=host --entrypoint stat --init -e CONTAINER_IMAGE=docker.io/prom/prometheus:v2.18.1 -e NODE_NAME=smithi040 -e CEPH_USE_RANDOM_NONCE=1 docker.io/prom/prometheus:v2.18.1 -c %u %g /etc/prometheus 2023-11-17T05:31:07.831 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:31:07 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: stat: stderr Trying to pull docker.io/prom/prometheus:v2.18.1... 2023-11-17T05:31:07.831 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:31:07 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: stat: stderr time="2023-11-17T05:29:15Z" level=warning msg="Failed, retrying in 1s ... (1/3). Error: initializing source docker://prom/prometheus:v2.18.1: (Mirrors also failed: [docker-mirror.front.sepia.ceph.com:5000/prom/prometheus:v2.18.1: pinging container registry docker-mirror.front.sepia.ceph.com:5000: Get \"https://docker-mirror.front.sepia.ceph.com:5000/v2/\": dial tcp 172.21.0.79:5000: connect: no route to host]): docker.io/prom/prometheus:v2.18.1: reading manifest v2.18.1 in docker.io/prom/prometheus: toomanyrequests: You have reached your pull rate limit. You may increase the limit by authenticating and upgrading: https://www.docker.com/increase-rate-limit" 2023-11-17T05:31:07.832 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:31:07 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: stat: stderr time="2023-11-17T05:29:52Z" level=warning msg="Failed, retrying in 1s ... (2/3). Error: initializing source docker://prom/prometheus:v2.18.1: (Mirrors also failed: [docker-mirror.front.sepia.ceph.com:5000/prom/prometheus:v2.18.1: pinging container registry docker-mirror.front.sepia.ceph.com:5000: Get \"https://docker-mirror.front.sepia.ceph.com:5000/v2/\": dial tcp 172.21.0.79:5000: connect: no route to host]): docker.io/prom/prometheus:v2.18.1: reading manifest v2.18.1 in docker.io/prom/prometheus: toomanyrequests: You have reached your pull rate limit. You may increase the limit by authenticating and upgrading: https://www.docker.com/increase-rate-limit" 2023-11-17T05:31:07.832 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:31:07 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: stat: stderr time="2023-11-17T05:30:29Z" level=warning msg="Failed, retrying in 1s ... (3/3). Error: initializing source docker://prom/prometheus:v2.18.1: (Mirrors also failed: [docker-mirror.front.sepia.ceph.com:5000/prom/prometheus:v2.18.1: pinging container registry docker-mirror.front.sepia.ceph.com:5000: Get \"https://docker-mirror.front.sepia.ceph.com:5000/v2/\": dial tcp 172.21.0.79:5000: connect: no route to host]): docker.io/prom/prometheus:v2.18.1: reading manifest v2.18.1 in docker.io/prom/prometheus: toomanyrequests: You have reached your pull rate limit. You may increase the limit by authenticating and upgrading: https://www.docker.com/increase-rate-limit" 2023-11-17T05:31:07.832 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:31:07 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: stat: stderr Error: initializing source docker://prom/prometheus:v2.18.1: (Mirrors also failed: [docker-mirror.front.sepia.ceph.com:5000/prom/prometheus:v2.18.1: pinging container registry docker-mirror.front.sepia.ceph.com:5000: Get "https://docker-mirror.front.sepia.ceph.com:5000/v2/": dial tcp 172.21.0.79:5000: connect: no route to host]): docker.io/prom/prometheus:v2.18.1: reading manifest v2.18.1 in docker.io/prom/prometheus: toomanyrequests: You have reached your pull rate limit. You may increase the limit by authenticating and upgrading: https://www.docker.com/increase-rate-limit 2023-11-17T05:31:07.832 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:31:07 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: Traceback (most recent call last): 2023-11-17T05:31:07.832 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:31:07 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: File "/var/lib/ceph/ed6470e2-8508-11ee-8dbe-212e2dc638e7/cephadm.30cb78bdbbafb384af862e1c2292b944f15942b586128e91262b43e91e11ae90", line 8029, in 2023-11-17T05:31:07.832 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:31:07 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: main() 2023-11-17T05:31:07.832 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:31:07 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: File "/var/lib/ceph/ed6470e2-8508-11ee-8dbe-212e2dc638e7/cephadm.30cb78bdbbafb384af862e1c2292b944f15942b586128e91262b43e91e11ae90", line 8017, in main 2023-11-17T05:31:07.832 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:31:07 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: r = ctx.func(ctx) 2023-11-17T05:31:07.832 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:31:07 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: File "/var/lib/ceph/ed6470e2-8508-11ee-8dbe-212e2dc638e7/cephadm.30cb78bdbbafb384af862e1c2292b944f15942b586128e91262b43e91e11ae90", line 1760, in _default_image 2023-11-17T05:31:07.832 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:31:07 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: return func(ctx) 2023-11-17T05:31:07.832 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:31:07 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: File "/var/lib/ceph/ed6470e2-8508-11ee-8dbe-212e2dc638e7/cephadm.30cb78bdbbafb384af862e1c2292b944f15942b586128e91262b43e91e11ae90", line 4263, in command_deploy 2023-11-17T05:31:07.833 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:31:07 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: uid, gid = extract_uid_gid_monitoring(ctx, daemon_type) 2023-11-17T05:31:07.833 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:31:07 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: File "/var/lib/ceph/ed6470e2-8508-11ee-8dbe-212e2dc638e7/cephadm.30cb78bdbbafb384af862e1c2292b944f15942b586128e91262b43e91e11ae90", line 4185, in extract_uid_gid_monitoring 2023-11-17T05:31:07.833 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:31:07 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: uid, gid = extract_uid_gid(ctx, file_path='/etc/prometheus') 2023-11-17T05:31:07.833 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:31:07 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: File "/var/lib/ceph/ed6470e2-8508-11ee-8dbe-212e2dc638e7/cephadm.30cb78bdbbafb384af862e1c2292b944f15942b586128e91262b43e91e11ae90", line 2531, in extract_uid_gid 2023-11-17T05:31:07.833 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:31:07 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: raise RuntimeError('uid/gid not found') 2023-11-17T05:31:07.833 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:31:07 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: RuntimeError: uid/gid not found 2023-11-17T05:31:07.833 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:31:07 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:31:07.251572+0000 mon.smithi040 (mon.0) 260 : audit [DBG] from='mgr.14164 172.21.15.40:0/2203032451' entity='mgr.smithi040.ogcprl' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-11-17T05:31:07.833 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:31:07 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:31:07.255674+0000 mgr.smithi040.ogcprl (mgr.14164) 307 : cluster [DBG] pgmap v292: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:31:08.516 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:08.515+0000 7fe8f8976700 1 -- 172.21.15.66:0/3188399369 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fe8f4060280 msgr2=0x7fe8f40fb4f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:31:08.517 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:08.515+0000 7fe8f8976700 1 --2- 172.21.15.66:0/3188399369 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fe8f4060280 0x7fe8f40fb4f0 secure :-1 s=READY pgs=319 cs=0 l=1 rev1=1 rx=0x7fe8e4009fb0 tx=0x7fe8e4009b30).stop 2023-11-17T05:31:08.517 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:08.515+0000 7fe8f8976700 1 -- 172.21.15.66:0/3188399369 shutdown_connections 2023-11-17T05:31:08.517 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:08.515+0000 7fe8f8976700 1 --2- 172.21.15.66:0/3188399369 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fe8f4060280 0x7fe8f40fb4f0 unknown :-1 s=CLOSED pgs=319 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:31:08.517 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:08.515+0000 7fe8f8976700 1 -- 172.21.15.66:0/3188399369 >> 172.21.15.66:0/3188399369 conn(0x7fe8f40f0b70 msgr2=0x7fe8f40f2f90 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:31:08.517 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:08.515+0000 7fe8f8976700 1 -- 172.21.15.66:0/3188399369 shutdown_connections 2023-11-17T05:31:08.517 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:08.515+0000 7fe8f8976700 1 -- 172.21.15.66:0/3188399369 wait complete. 2023-11-17T05:31:08.517 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:08.515+0000 7fe8f8976700 1 Processor -- start 2023-11-17T05:31:08.517 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:08.515+0000 7fe8f8976700 1 -- start start 2023-11-17T05:31:08.519 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:08.516+0000 7fe8f8976700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fe8f4060280 0x7fe8f41088e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:31:08.519 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:08.516+0000 7fe8f8976700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fe8f4108dd0 con 0x7fe8f4060280 2023-11-17T05:31:08.520 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:08.516+0000 7fe8f259c700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fe8f4060280 0x7fe8f41088e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:31:08.520 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:08.516+0000 7fe8f259c700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fe8f4060280 0x7fe8f41088e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.66:50966/0 (socket says 172.21.15.66:50966) 2023-11-17T05:31:08.520 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:08.516+0000 7fe8f259c700 1 -- 172.21.15.66:0/2551268760 learned_addr learned my addr 172.21.15.66:0/2551268760 (peer_addr_for_me v2:172.21.15.66:0/0) 2023-11-17T05:31:08.520 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:08.516+0000 7fe8f259c700 1 -- 172.21.15.66:0/2551268760 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fe8e4005040 con 0x7fe8f4060280 2023-11-17T05:31:08.520 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:08.516+0000 7fe8f259c700 1 --2- 172.21.15.66:0/2551268760 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fe8f4060280 0x7fe8f41088e0 secure :-1 s=READY pgs=320 cs=0 l=1 rev1=1 rx=0x7fe8e4013510 tx=0x7fe8e400b040).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:31:08.520 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:08.516+0000 7fe8e2ffd700 1 -- 172.21.15.66:0/2551268760 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7fe8e4009320 con 0x7fe8f4060280 2023-11-17T05:31:08.520 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:08.516+0000 7fe8e2ffd700 1 -- 172.21.15.66:0/2551268760 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(25 keys) v1 ==== 1074+0+0 (secure 0 0 0) 0x7fe8e4013f00 con 0x7fe8f4060280 2023-11-17T05:31:08.520 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:08.516+0000 7fe8e2ffd700 1 -- 172.21.15.66:0/2551268760 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7fe8e4014dc0 con 0x7fe8f4060280 2023-11-17T05:31:08.520 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:08.516+0000 7fe8f8976700 1 -- 172.21.15.66:0/2551268760 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fe8f4108fd0 con 0x7fe8f4060280 2023-11-17T05:31:08.520 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:08.517+0000 7fe8f8976700 1 -- 172.21.15.66:0/2551268760 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fe8f4109460 con 0x7fe8f4060280 2023-11-17T05:31:08.520 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:08.517+0000 7fe8e2ffd700 1 -- 172.21.15.66:0/2551268760 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 17) v1 ==== 36269+0+0 (secure 0 0 0) 0x7fe8e4014420 con 0x7fe8f4060280 2023-11-17T05:31:08.521 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:08.517+0000 7fe8e2ffd700 1 --2- 172.21.15.66:0/2551268760 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fe8dc02ffb0 0x7fe8dc032470 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:31:08.521 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:08.517+0000 7fe8e2ffd700 1 -- 172.21.15.66:0/2551268760 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(5..5 src has 1..5) v4 ==== 1270+0+0 (secure 0 0 0) 0x7fe8e40410b0 con 0x7fe8f4060280 2023-11-17T05:31:08.521 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:08.517+0000 7fe8f8976700 1 -- 172.21.15.66:0/2551268760 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fe8f4045ad0 con 0x7fe8f4060280 2023-11-17T05:31:08.521 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:08.518+0000 7fe8f1d9b700 1 --2- 172.21.15.66:0/2551268760 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fe8dc02ffb0 0x7fe8dc032470 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:31:08.521 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:08.518+0000 7fe8f1d9b700 1 --2- 172.21.15.66:0/2551268760 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fe8dc02ffb0 0x7fe8dc032470 secure :-1 s=READY pgs=125 cs=0 l=1 rev1=1 rx=0x7fe8e80096a0 tx=0x7fe8e8006b40).ready entity=mgr.14164 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:31:08.523 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:08.522+0000 7fe8e2ffd700 1 -- 172.21.15.66:0/2551268760 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+152212 (secure 0 0 0) 0x7fe8e40095f0 con 0x7fe8f4060280 2023-11-17T05:31:08.738 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:31:08 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:31:07.556216+0000 mon.smithi040 (mon.0) 261 : audit [INF] from='mgr.14164 172.21.15.40:0/2203032451' entity='mgr.smithi040.ogcprl' 2023-11-17T05:31:08.739 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:31:08 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:31:07.894321+0000 mon.smithi040 (mon.0) 262 : audit [INF] from='mgr.14164 172.21.15.40:0/2203032451' entity='mgr.smithi040.ogcprl' 2023-11-17T05:31:08.751 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:08.749+0000 7fe8f8976700 1 -- 172.21.15.66:0/2551268760 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "mon dump", "format": "json"} v 0) v1 -- 0x7fe8f410a2a0 con 0x7fe8f4060280 2023-11-17T05:31:08.752 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:08.750+0000 7fe8e2ffd700 1 -- 172.21.15.66:0/2551268760 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mon_command_ack([{"prefix": "mon dump", "format": "json"}]=0 dumped monmap epoch 1 v1) v1 ==== 95+0+690 (secure 0 0 0) 0x7fe8e401db70 con 0x7fe8f4060280 2023-11-17T05:31:08.752 INFO:teuthology.orchestra.run.smithi066.stdout: 2023-11-17T05:31:08.752 INFO:teuthology.orchestra.run.smithi066.stdout:{"epoch":1,"fsid":"ed6470e2-8508-11ee-8dbe-212e2dc638e7","modified":"2023-11-17T05:20:13.934038Z","created":"2023-11-17T05:20:13.934038Z","min_mon_release":16,"min_mon_release_name":"pacific","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging"],"optional":[]},"mons":[{"rank":0,"name":"smithi040","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:3300","nonce":0},{"type":"v1","addr":"172.21.15.40:6789","nonce":0}]},"addr":"172.21.15.40:6789/0","public_addr":"172.21.15.40:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2023-11-17T05:31:08.754 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:08.752+0000 7fe8f8976700 1 -- 172.21.15.66:0/2551268760 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fe8dc02ffb0 msgr2=0x7fe8dc032470 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:31:08.754 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:08.752+0000 7fe8f8976700 1 --2- 172.21.15.66:0/2551268760 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fe8dc02ffb0 0x7fe8dc032470 secure :-1 s=READY pgs=125 cs=0 l=1 rev1=1 rx=0x7fe8e80096a0 tx=0x7fe8e8006b40).stop 2023-11-17T05:31:08.754 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:08.752+0000 7fe8f8976700 1 -- 172.21.15.66:0/2551268760 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fe8f4060280 msgr2=0x7fe8f41088e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:31:08.754 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:08.752+0000 7fe8f8976700 1 --2- 172.21.15.66:0/2551268760 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fe8f4060280 0x7fe8f41088e0 secure :-1 s=READY pgs=320 cs=0 l=1 rev1=1 rx=0x7fe8e4013510 tx=0x7fe8e400b040).stop 2023-11-17T05:31:08.754 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:08.752+0000 7fe8f8976700 1 -- 172.21.15.66:0/2551268760 shutdown_connections 2023-11-17T05:31:08.754 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:08.752+0000 7fe8f8976700 1 --2- 172.21.15.66:0/2551268760 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fe8dc02ffb0 0x7fe8dc032470 unknown :-1 s=CLOSED pgs=125 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:31:08.755 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:08.752+0000 7fe8f8976700 1 --2- 172.21.15.66:0/2551268760 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fe8f4060280 0x7fe8f41088e0 unknown :-1 s=CLOSED pgs=320 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:31:08.755 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:08.752+0000 7fe8f8976700 1 -- 172.21.15.66:0/2551268760 >> 172.21.15.66:0/2551268760 conn(0x7fe8f40f0b70 msgr2=0x7fe8f40f1660 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:31:08.755 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:08.752+0000 7fe8f8976700 1 -- 172.21.15.66:0/2551268760 shutdown_connections 2023-11-17T05:31:08.755 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:08.752+0000 7fe8f8976700 1 -- 172.21.15.66:0/2551268760 wait complete. 2023-11-17T05:31:08.755 INFO:teuthology.orchestra.run.smithi066.stderr:dumped monmap epoch 1 2023-11-17T05:31:09.989 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:31:09 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:31:08.584224+0000 mon.smithi040 (mon.0) 263 : audit [INF] from='mgr.14164 172.21.15.40:0/2203032451' entity='mgr.smithi040.ogcprl' 2023-11-17T05:31:09.989 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:31:09 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:31:08.751783+0000 mon.smithi040 (mon.0) 264 : audit [DBG] from='client.? 172.21.15.66:0/2551268760' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-11-17T05:31:09.989 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:31:09 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:31:09.255936+0000 mgr.smithi040.ogcprl (mgr.14164) 308 : cluster [DBG] pgmap v293: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:31:10.445 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2023-11-17T05:31:10.445 DEBUG:teuthology.orchestra.run.smithi066:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.4 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ed6470e2-8508-11ee-8dbe-212e2dc638e7 -- ceph mon dump -f json 2023-11-17T05:31:11.633 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:11.631+0000 7f3830eb1700 1 -- 172.21.15.66:0/1566205895 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f382c060180 msgr2=0x7f382c0fb380 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:31:11.633 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:11.631+0000 7f3830eb1700 1 --2- 172.21.15.66:0/1566205895 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f382c060180 0x7f382c0fb380 secure :-1 s=READY pgs=321 cs=0 l=1 rev1=1 rx=0x7f381c009fb0 tx=0x7f381c009b30).stop 2023-11-17T05:31:11.633 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:11.631+0000 7f3830eb1700 1 -- 172.21.15.66:0/1566205895 shutdown_connections 2023-11-17T05:31:11.633 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:11.631+0000 7f3830eb1700 1 --2- 172.21.15.66:0/1566205895 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f382c060180 0x7f382c0fb380 unknown :-1 s=CLOSED pgs=321 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:31:11.633 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:11.631+0000 7f3830eb1700 1 -- 172.21.15.66:0/1566205895 >> 172.21.15.66:0/1566205895 conn(0x7f382c0f09a0 msgr2=0x7f382c0f2dc0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:31:11.634 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:11.631+0000 7f3830eb1700 1 -- 172.21.15.66:0/1566205895 shutdown_connections 2023-11-17T05:31:11.634 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:11.631+0000 7f3830eb1700 1 -- 172.21.15.66:0/1566205895 wait complete. 2023-11-17T05:31:11.634 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:11.631+0000 7f3830eb1700 1 Processor -- start 2023-11-17T05:31:11.634 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:11.632+0000 7f3830eb1700 1 -- start start 2023-11-17T05:31:11.634 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:11.632+0000 7f3830eb1700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f382c060180 0x7f382c1086d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:31:11.634 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:11.632+0000 7f3830eb1700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f382c108bc0 con 0x7f382c060180 2023-11-17T05:31:11.636 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:11.632+0000 7f382b7fe700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f382c060180 0x7f382c1086d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:31:11.636 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:11.632+0000 7f382b7fe700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f382c060180 0x7f382c1086d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.66:50992/0 (socket says 172.21.15.66:50992) 2023-11-17T05:31:11.636 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:11.632+0000 7f382b7fe700 1 -- 172.21.15.66:0/2918821462 learned_addr learned my addr 172.21.15.66:0/2918821462 (peer_addr_for_me v2:172.21.15.66:0/0) 2023-11-17T05:31:11.636 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:11.633+0000 7f382b7fe700 1 -- 172.21.15.66:0/2918821462 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f381c005040 con 0x7f382c060180 2023-11-17T05:31:11.636 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:11.633+0000 7f382b7fe700 1 --2- 172.21.15.66:0/2918821462 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f382c060180 0x7f382c1086d0 secure :-1 s=READY pgs=322 cs=0 l=1 rev1=1 rx=0x7f381c013510 tx=0x7f381c00b040).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:31:11.636 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:11.633+0000 7f3813fff700 1 -- 172.21.15.66:0/2918821462 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f381c009320 con 0x7f382c060180 2023-11-17T05:31:11.636 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:11.633+0000 7f3830eb1700 1 -- 172.21.15.66:0/2918821462 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f382c108dc0 con 0x7f382c060180 2023-11-17T05:31:11.636 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:11.633+0000 7f3813fff700 1 -- 172.21.15.66:0/2918821462 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(25 keys) v1 ==== 1074+0+0 (secure 0 0 0) 0x7f381c013f00 con 0x7f382c060180 2023-11-17T05:31:11.636 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:11.633+0000 7f3813fff700 1 -- 172.21.15.66:0/2918821462 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f381c014d90 con 0x7f382c060180 2023-11-17T05:31:11.636 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:11.633+0000 7f3830eb1700 1 -- 172.21.15.66:0/2918821462 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f382c109250 con 0x7f382c060180 2023-11-17T05:31:11.636 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:11.634+0000 7f3813fff700 1 -- 172.21.15.66:0/2918821462 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 17) v1 ==== 36269+0+0 (secure 0 0 0) 0x7f381c0263a0 con 0x7f382c060180 2023-11-17T05:31:11.637 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:11.634+0000 7f3830eb1700 1 -- 172.21.15.66:0/2918821462 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f382c045ad0 con 0x7f382c060180 2023-11-17T05:31:11.637 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:11.634+0000 7f3813fff700 1 --2- 172.21.15.66:0/2918821462 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f381402ff60 0x7f3814032420 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:31:11.637 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:11.634+0000 7f3813fff700 1 -- 172.21.15.66:0/2918821462 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(5..5 src has 1..5) v4 ==== 1270+0+0 (secure 0 0 0) 0x7f381c0410c0 con 0x7f382c060180 2023-11-17T05:31:11.637 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:11.634+0000 7f382affd700 1 --2- 172.21.15.66:0/2918821462 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f381402ff60 0x7f3814032420 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:31:11.637 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:11.635+0000 7f382affd700 1 --2- 172.21.15.66:0/2918821462 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f381402ff60 0x7f3814032420 secure :-1 s=READY pgs=126 cs=0 l=1 rev1=1 rx=0x7f38200096a0 tx=0x7f3820006b40).ready entity=mgr.14164 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:31:11.640 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:11.639+0000 7f3813fff700 1 -- 172.21.15.66:0/2918821462 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+152212 (secure 0 0 0) 0x7f381c0126f0 con 0x7f382c060180 2023-11-17T05:31:11.877 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:11.876+0000 7f3830eb1700 1 -- 172.21.15.66:0/2918821462 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "mon dump", "format": "json"} v 0) v1 -- 0x7f382c10a100 con 0x7f382c060180 2023-11-17T05:31:11.878 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:11.877+0000 7f3813fff700 1 -- 172.21.15.66:0/2918821462 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mon_command_ack([{"prefix": "mon dump", "format": "json"}]=0 dumped monmap epoch 1 v1) v1 ==== 95+0+690 (secure 0 0 0) 0x7f381c012330 con 0x7f382c060180 2023-11-17T05:31:11.879 INFO:teuthology.orchestra.run.smithi066.stdout: 2023-11-17T05:31:11.879 INFO:teuthology.orchestra.run.smithi066.stdout:{"epoch":1,"fsid":"ed6470e2-8508-11ee-8dbe-212e2dc638e7","modified":"2023-11-17T05:20:13.934038Z","created":"2023-11-17T05:20:13.934038Z","min_mon_release":16,"min_mon_release_name":"pacific","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging"],"optional":[]},"mons":[{"rank":0,"name":"smithi040","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:3300","nonce":0},{"type":"v1","addr":"172.21.15.40:6789","nonce":0}]},"addr":"172.21.15.40:6789/0","public_addr":"172.21.15.40:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2023-11-17T05:31:11.880 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:11.878+0000 7f3830eb1700 1 -- 172.21.15.66:0/2918821462 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f381402ff60 msgr2=0x7f3814032420 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:31:11.880 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:11.878+0000 7f3830eb1700 1 --2- 172.21.15.66:0/2918821462 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f381402ff60 0x7f3814032420 secure :-1 s=READY pgs=126 cs=0 l=1 rev1=1 rx=0x7f38200096a0 tx=0x7f3820006b40).stop 2023-11-17T05:31:11.880 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:11.878+0000 7f3830eb1700 1 -- 172.21.15.66:0/2918821462 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f382c060180 msgr2=0x7f382c1086d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:31:11.881 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:11.878+0000 7f3830eb1700 1 --2- 172.21.15.66:0/2918821462 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f382c060180 0x7f382c1086d0 secure :-1 s=READY pgs=322 cs=0 l=1 rev1=1 rx=0x7f381c013510 tx=0x7f381c00b040).stop 2023-11-17T05:31:11.881 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:11.879+0000 7f3830eb1700 1 -- 172.21.15.66:0/2918821462 shutdown_connections 2023-11-17T05:31:11.881 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:11.879+0000 7f3830eb1700 1 --2- 172.21.15.66:0/2918821462 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f381402ff60 0x7f3814032420 unknown :-1 s=CLOSED pgs=126 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:31:11.881 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:11.879+0000 7f3830eb1700 1 --2- 172.21.15.66:0/2918821462 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f382c060180 0x7f382c1086d0 unknown :-1 s=CLOSED pgs=322 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:31:11.881 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:11.879+0000 7f3830eb1700 1 -- 172.21.15.66:0/2918821462 >> 172.21.15.66:0/2918821462 conn(0x7f382c0f09a0 msgr2=0x7f382c0f14d0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:31:11.881 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:11.879+0000 7f3830eb1700 1 -- 172.21.15.66:0/2918821462 shutdown_connections 2023-11-17T05:31:11.881 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:11.879+0000 7f3830eb1700 1 -- 172.21.15.66:0/2918821462 wait complete. 2023-11-17T05:31:11.881 INFO:teuthology.orchestra.run.smithi066.stderr:dumped monmap epoch 1 2023-11-17T05:31:12.239 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:31:11 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:31:10.737872+0000 mon.smithi040 (mon.0) 265 : audit [INF] from='mgr.14164 172.21.15.40:0/2203032451' entity='mgr.smithi040.ogcprl' 2023-11-17T05:31:12.239 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:31:11 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:31:11.256291+0000 mgr.smithi040.ogcprl (mgr.14164) 309 : cluster [DBG] pgmap v294: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:31:12.239 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:31:11 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:31:11.318272+0000 mon.smithi040 (mon.0) 266 : audit [INF] from='mgr.14164 172.21.15.40:0/2203032451' entity='mgr.smithi040.ogcprl' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi040.ogcprl/mirror_snapshot_schedule"}]: dispatch 2023-11-17T05:31:12.239 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:31:11 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:31:11.318587+0000 mon.smithi040 (mon.0) 267 : audit [INF] from='mgr.14164 172.21.15.40:0/2203032451' entity='mgr.smithi040.ogcprl' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi040.ogcprl/trash_purge_schedule"}]: dispatch 2023-11-17T05:31:12.239 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:31:11 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:31:11.511769+0000 mon.smithi040 (mon.0) 268 : audit [INF] from='mgr.14164 172.21.15.40:0/2203032451' entity='mgr.smithi040.ogcprl' 2023-11-17T05:31:12.239 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:31:11 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:31:11.512601+0000 mgr.smithi040.ogcprl (mgr.14164) 310 : cluster [DBG] pgmap v295: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:31:12.239 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:31:11 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:31:11.514912+0000 mon.smithi040 (mon.0) 269 : audit [INF] from='mgr.14164 172.21.15.40:0/2203032451' entity='mgr.smithi040.ogcprl' 2023-11-17T05:31:12.239 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:31:11 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cephadm 2023-11-17T05:31:11.517068+0000 mgr.smithi040.ogcprl (mgr.14164) 311 : cephadm [INF] Deploying daemon alertmanager.smithi066 on smithi066 2023-11-17T05:31:13.239 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:31:12 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:31:11.735955+0000 mon.smithi040 (mon.0) 270 : cluster [WRN] Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2023-11-17T05:31:13.326 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:31:12 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:31:11.878369+0000 mon.smithi040 (mon.0) 271 : audit [DBG] from='client.? 172.21.15.66:0/2918821462' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-11-17T05:31:13.326 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2023-11-17T05:31:13.327 DEBUG:teuthology.orchestra.run.smithi066:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.4 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ed6470e2-8508-11ee-8dbe-212e2dc638e7 -- ceph mon dump -f json 2023-11-17T05:31:14.238 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:31:13 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:31:13.512917+0000 mgr.smithi040.ogcprl (mgr.14164) 312 : cluster [DBG] pgmap v296: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:31:14.471 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:14.469+0000 7f2198bcd700 1 -- 172.21.15.66:0/4246799773 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f21940f51a0 msgr2=0x7f21940f55c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:31:14.471 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:14.469+0000 7f2198bcd700 1 --2- 172.21.15.66:0/4246799773 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f21940f51a0 0x7f21940f55c0 secure :-1 s=READY pgs=323 cs=0 l=1 rev1=1 rx=0x7f2184009fb0 tx=0x7f2184009b30).stop 2023-11-17T05:31:14.471 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:14.469+0000 7f2198bcd700 1 -- 172.21.15.66:0/4246799773 shutdown_connections 2023-11-17T05:31:14.471 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:14.469+0000 7f2198bcd700 1 --2- 172.21.15.66:0/4246799773 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f21940f51a0 0x7f21940f55c0 unknown :-1 s=CLOSED pgs=323 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:31:14.471 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:14.469+0000 7f2198bcd700 1 -- 172.21.15.66:0/4246799773 >> 172.21.15.66:0/4246799773 conn(0x7f21940f09a0 msgr2=0x7f21940f2dc0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:31:14.471 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:14.469+0000 7f2198bcd700 1 -- 172.21.15.66:0/4246799773 shutdown_connections 2023-11-17T05:31:14.472 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:14.469+0000 7f2198bcd700 1 -- 172.21.15.66:0/4246799773 wait complete. 2023-11-17T05:31:14.472 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:14.469+0000 7f2198bcd700 1 Processor -- start 2023-11-17T05:31:14.472 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:14.470+0000 7f2198bcd700 1 -- start start 2023-11-17T05:31:14.472 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:14.470+0000 7f2198bcd700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f21940601e0 0x7f2194109360 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:31:14.474 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:14.470+0000 7f2198bcd700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f2194109850 con 0x7f21940601e0 2023-11-17T05:31:14.474 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:14.470+0000 7f21937fe700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f21940601e0 0x7f2194109360 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:31:14.474 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:14.470+0000 7f21937fe700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f21940601e0 0x7f2194109360 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.66:43520/0 (socket says 172.21.15.66:43520) 2023-11-17T05:31:14.474 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:14.470+0000 7f21937fe700 1 -- 172.21.15.66:0/1617136115 learned_addr learned my addr 172.21.15.66:0/1617136115 (peer_addr_for_me v2:172.21.15.66:0/0) 2023-11-17T05:31:14.474 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:14.470+0000 7f21937fe700 1 -- 172.21.15.66:0/1617136115 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f2184005040 con 0x7f21940601e0 2023-11-17T05:31:14.474 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:14.470+0000 7f21937fe700 1 --2- 172.21.15.66:0/1617136115 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f21940601e0 0x7f2194109360 secure :-1 s=READY pgs=324 cs=0 l=1 rev1=1 rx=0x7f2184009f80 tx=0x7f2184009b30).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:31:14.474 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:14.471+0000 7f217bfff700 1 -- 172.21.15.66:0/1617136115 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f21840136c0 con 0x7f21940601e0 2023-11-17T05:31:14.474 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:14.471+0000 7f217bfff700 1 -- 172.21.15.66:0/1617136115 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(25 keys) v1 ==== 1074+0+0 (secure 0 0 0) 0x7f21840095f0 con 0x7f21940601e0 2023-11-17T05:31:14.474 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:14.471+0000 7f2198bcd700 1 -- 172.21.15.66:0/1617136115 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f2194109990 con 0x7f21940601e0 2023-11-17T05:31:14.475 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:14.471+0000 7f217bfff700 1 -- 172.21.15.66:0/1617136115 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f2184014420 con 0x7f21940601e0 2023-11-17T05:31:14.475 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:14.471+0000 7f2198bcd700 1 -- 172.21.15.66:0/1617136115 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f2194109dc0 con 0x7f21940601e0 2023-11-17T05:31:14.475 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:14.471+0000 7f217bfff700 1 -- 172.21.15.66:0/1617136115 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 17) v1 ==== 36269+0+0 (secure 0 0 0) 0x7f2184025060 con 0x7f21940601e0 2023-11-17T05:31:14.475 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:14.472+0000 7f2198bcd700 1 -- 172.21.15.66:0/1617136115 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f2194045ad0 con 0x7f21940601e0 2023-11-17T05:31:14.475 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:14.472+0000 7f217bfff700 1 --2- 172.21.15.66:0/1617136115 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f217c02ff60 0x7f217c032420 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:31:14.475 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:14.472+0000 7f217bfff700 1 -- 172.21.15.66:0/1617136115 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(5..5 src has 1..5) v4 ==== 1270+0+0 (secure 0 0 0) 0x7f2184040db0 con 0x7f21940601e0 2023-11-17T05:31:14.475 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:14.472+0000 7f2192ffd700 1 --2- 172.21.15.66:0/1617136115 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f217c02ff60 0x7f217c032420 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:31:14.475 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:14.472+0000 7f2192ffd700 1 --2- 172.21.15.66:0/1617136115 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f217c02ff60 0x7f217c032420 secure :-1 s=READY pgs=127 cs=0 l=1 rev1=1 rx=0x7f21880096a0 tx=0x7f2188006b40).ready entity=mgr.14164 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:31:14.478 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:14.477+0000 7f217bfff700 1 -- 172.21.15.66:0/1617136115 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+152212 (secure 0 0 0) 0x7f2184004490 con 0x7f21940601e0 2023-11-17T05:31:14.703 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:14.697+0000 7f2198bcd700 1 -- 172.21.15.66:0/1617136115 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "mon dump", "format": "json"} v 0) v1 -- 0x7f21940fabe0 con 0x7f21940601e0 2023-11-17T05:31:14.703 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:14.697+0000 7f217bfff700 1 -- 172.21.15.66:0/1617136115 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mon_command_ack([{"prefix": "mon dump", "format": "json"}]=0 dumped monmap epoch 1 v1) v1 ==== 95+0+690 (secure 0 0 0) 0x7f218402d2d0 con 0x7f21940601e0 2023-11-17T05:31:14.704 INFO:teuthology.orchestra.run.smithi066.stdout: 2023-11-17T05:31:14.704 INFO:teuthology.orchestra.run.smithi066.stdout:{"epoch":1,"fsid":"ed6470e2-8508-11ee-8dbe-212e2dc638e7","modified":"2023-11-17T05:20:13.934038Z","created":"2023-11-17T05:20:13.934038Z","min_mon_release":16,"min_mon_release_name":"pacific","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging"],"optional":[]},"mons":[{"rank":0,"name":"smithi040","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:3300","nonce":0},{"type":"v1","addr":"172.21.15.40:6789","nonce":0}]},"addr":"172.21.15.40:6789/0","public_addr":"172.21.15.40:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2023-11-17T05:31:14.706 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:14.699+0000 7f2198bcd700 1 -- 172.21.15.66:0/1617136115 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f217c02ff60 msgr2=0x7f217c032420 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:31:14.707 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:14.699+0000 7f2198bcd700 1 --2- 172.21.15.66:0/1617136115 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f217c02ff60 0x7f217c032420 secure :-1 s=READY pgs=127 cs=0 l=1 rev1=1 rx=0x7f21880096a0 tx=0x7f2188006b40).stop 2023-11-17T05:31:14.707 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:14.699+0000 7f2198bcd700 1 -- 172.21.15.66:0/1617136115 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f21940601e0 msgr2=0x7f2194109360 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:31:14.707 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:14.699+0000 7f2198bcd700 1 --2- 172.21.15.66:0/1617136115 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f21940601e0 0x7f2194109360 secure :-1 s=READY pgs=324 cs=0 l=1 rev1=1 rx=0x7f2184009f80 tx=0x7f2184009b30).stop 2023-11-17T05:31:14.707 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:14.699+0000 7f2198bcd700 1 -- 172.21.15.66:0/1617136115 shutdown_connections 2023-11-17T05:31:14.707 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:14.699+0000 7f2198bcd700 1 --2- 172.21.15.66:0/1617136115 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f217c02ff60 0x7f217c032420 unknown :-1 s=CLOSED pgs=127 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:31:14.707 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:14.699+0000 7f2198bcd700 1 --2- 172.21.15.66:0/1617136115 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f21940601e0 0x7f2194109360 unknown :-1 s=CLOSED pgs=324 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:31:14.707 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:14.699+0000 7f2198bcd700 1 -- 172.21.15.66:0/1617136115 >> 172.21.15.66:0/1617136115 conn(0x7f21940f09a0 msgr2=0x7f21940f1500 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:31:14.707 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:14.699+0000 7f2198bcd700 1 -- 172.21.15.66:0/1617136115 shutdown_connections 2023-11-17T05:31:14.707 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:14.699+0000 7f2198bcd700 1 -- 172.21.15.66:0/1617136115 wait complete. 2023-11-17T05:31:14.707 INFO:teuthology.orchestra.run.smithi066.stderr:dumped monmap epoch 1 2023-11-17T05:31:15.738 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:31:15 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:31:14.699031+0000 mon.smithi040 (mon.0) 272 : audit [DBG] from='client.? 172.21.15.66:0/1617136115' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-11-17T05:31:16.074 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2023-11-17T05:31:16.074 DEBUG:teuthology.orchestra.run.smithi066:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.4 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ed6470e2-8508-11ee-8dbe-212e2dc638e7 -- ceph mon dump -f json 2023-11-17T05:31:16.989 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:31:16 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:31:15.513251+0000 mgr.smithi040.ogcprl (mgr.14164) 313 : cluster [DBG] pgmap v297: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:31:17.273 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:17.271+0000 7f957f59e700 1 -- 172.21.15.66:0/4216980997 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f95800f51a0 msgr2=0x7f95800f55c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:31:17.273 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:17.271+0000 7f957f59e700 1 --2- 172.21.15.66:0/4216980997 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f95800f51a0 0x7f95800f55c0 secure :-1 s=READY pgs=325 cs=0 l=1 rev1=1 rx=0x7f9570009fb0 tx=0x7f9570009b30).stop 2023-11-17T05:31:17.273 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:17.271+0000 7f957f59e700 1 -- 172.21.15.66:0/4216980997 shutdown_connections 2023-11-17T05:31:17.274 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:17.271+0000 7f957f59e700 1 --2- 172.21.15.66:0/4216980997 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f95800f51a0 0x7f95800f55c0 unknown :-1 s=CLOSED pgs=325 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:31:17.274 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:17.271+0000 7f957f59e700 1 -- 172.21.15.66:0/4216980997 >> 172.21.15.66:0/4216980997 conn(0x7f95800f09a0 msgr2=0x7f95800f2dc0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:31:17.274 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:17.271+0000 7f957f59e700 1 -- 172.21.15.66:0/4216980997 shutdown_connections 2023-11-17T05:31:17.274 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:17.271+0000 7f957f59e700 1 -- 172.21.15.66:0/4216980997 wait complete. 2023-11-17T05:31:17.274 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:17.271+0000 7f957f59e700 1 Processor -- start 2023-11-17T05:31:17.274 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:17.271+0000 7f957f59e700 1 -- start start 2023-11-17T05:31:17.274 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:17.272+0000 7f957f59e700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f95800f51a0 0x7f9580109170 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:31:17.274 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:17.272+0000 7f957f59e700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f9580060280 con 0x7f95800f51a0 2023-11-17T05:31:17.274 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:17.272+0000 7f957e59c700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f95800f51a0 0x7f9580109170 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:31:17.274 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:17.272+0000 7f957e59c700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f95800f51a0 0x7f9580109170 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.66:43544/0 (socket says 172.21.15.66:43544) 2023-11-17T05:31:17.274 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:17.272+0000 7f957e59c700 1 -- 172.21.15.66:0/4269881669 learned_addr learned my addr 172.21.15.66:0/4269881669 (peer_addr_for_me v2:172.21.15.66:0/0) 2023-11-17T05:31:17.275 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:17.272+0000 7f957e59c700 1 -- 172.21.15.66:0/4269881669 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f9570005040 con 0x7f95800f51a0 2023-11-17T05:31:17.275 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:17.272+0000 7f957e59c700 1 --2- 172.21.15.66:0/4269881669 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f95800f51a0 0x7f9580109170 secure :-1 s=READY pgs=326 cs=0 l=1 rev1=1 rx=0x7f9570013510 tx=0x7f957000b040).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:31:17.276 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:17.272+0000 7f956effd700 1 -- 172.21.15.66:0/4269881669 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f9570009320 con 0x7f95800f51a0 2023-11-17T05:31:17.276 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:17.272+0000 7f957f59e700 1 -- 172.21.15.66:0/4269881669 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f9580060480 con 0x7f95800f51a0 2023-11-17T05:31:17.276 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:17.272+0000 7f956effd700 1 -- 172.21.15.66:0/4269881669 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(25 keys) v1 ==== 1074+0+0 (secure 0 0 0) 0x7f9570013f00 con 0x7f95800f51a0 2023-11-17T05:31:17.276 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:17.272+0000 7f956effd700 1 -- 172.21.15.66:0/4269881669 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f9570014dc0 con 0x7f95800f51a0 2023-11-17T05:31:17.276 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:17.272+0000 7f957f59e700 1 -- 172.21.15.66:0/4269881669 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f95801098a0 con 0x7f95800f51a0 2023-11-17T05:31:17.276 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:17.273+0000 7f957f59e700 1 -- 172.21.15.66:0/4269881669 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f9580045ad0 con 0x7f95800f51a0 2023-11-17T05:31:17.276 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:17.273+0000 7f956effd700 1 -- 172.21.15.66:0/4269881669 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 17) v1 ==== 36269+0+0 (secure 0 0 0) 0x7f95700095f0 con 0x7f95800f51a0 2023-11-17T05:31:17.276 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:17.273+0000 7f956effd700 1 --2- 172.21.15.66:0/4269881669 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f956802ff60 0x7f9568032420 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:31:17.277 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:17.273+0000 7f956effd700 1 -- 172.21.15.66:0/4269881669 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(5..5 src has 1..5) v4 ==== 1270+0+0 (secure 0 0 0) 0x7f9570041020 con 0x7f95800f51a0 2023-11-17T05:31:17.277 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:17.274+0000 7f957dd9b700 1 --2- 172.21.15.66:0/4269881669 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f956802ff60 0x7f9568032420 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:31:17.277 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:17.274+0000 7f957dd9b700 1 --2- 172.21.15.66:0/4269881669 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f956802ff60 0x7f9568032420 secure :-1 s=READY pgs=128 cs=0 l=1 rev1=1 rx=0x7f95740096a0 tx=0x7f9574006b40).ready entity=mgr.14164 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:31:17.279 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:17.278+0000 7f956effd700 1 -- 172.21.15.66:0/4269881669 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+152212 (secure 0 0 0) 0x7f9570014420 con 0x7f95800f51a0 2023-11-17T05:31:17.499 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:17.498+0000 7f957f59e700 1 -- 172.21.15.66:0/4269881669 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "mon dump", "format": "json"} v 0) v1 -- 0x7f95800fabe0 con 0x7f95800f51a0 2023-11-17T05:31:17.500 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:17.499+0000 7f956effd700 1 -- 172.21.15.66:0/4269881669 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mon_command_ack([{"prefix": "mon dump", "format": "json"}]=0 dumped monmap epoch 1 v1) v1 ==== 95+0+690 (secure 0 0 0) 0x7f957001c920 con 0x7f95800f51a0 2023-11-17T05:31:17.500 INFO:teuthology.orchestra.run.smithi066.stdout: 2023-11-17T05:31:17.500 INFO:teuthology.orchestra.run.smithi066.stdout:{"epoch":1,"fsid":"ed6470e2-8508-11ee-8dbe-212e2dc638e7","modified":"2023-11-17T05:20:13.934038Z","created":"2023-11-17T05:20:13.934038Z","min_mon_release":16,"min_mon_release_name":"pacific","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging"],"optional":[]},"mons":[{"rank":0,"name":"smithi040","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:3300","nonce":0},{"type":"v1","addr":"172.21.15.40:6789","nonce":0}]},"addr":"172.21.15.40:6789/0","public_addr":"172.21.15.40:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2023-11-17T05:31:17.502 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:17.501+0000 7f957f59e700 1 -- 172.21.15.66:0/4269881669 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f956802ff60 msgr2=0x7f9568032420 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:31:17.502 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:17.501+0000 7f957f59e700 1 --2- 172.21.15.66:0/4269881669 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f956802ff60 0x7f9568032420 secure :-1 s=READY pgs=128 cs=0 l=1 rev1=1 rx=0x7f95740096a0 tx=0x7f9574006b40).stop 2023-11-17T05:31:17.503 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:17.501+0000 7f957f59e700 1 -- 172.21.15.66:0/4269881669 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f95800f51a0 msgr2=0x7f9580109170 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:31:17.503 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:17.501+0000 7f957f59e700 1 --2- 172.21.15.66:0/4269881669 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f95800f51a0 0x7f9580109170 secure :-1 s=READY pgs=326 cs=0 l=1 rev1=1 rx=0x7f9570013510 tx=0x7f957000b040).stop 2023-11-17T05:31:17.503 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:17.501+0000 7f957f59e700 1 -- 172.21.15.66:0/4269881669 shutdown_connections 2023-11-17T05:31:17.503 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:17.501+0000 7f957f59e700 1 --2- 172.21.15.66:0/4269881669 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f956802ff60 0x7f9568032420 unknown :-1 s=CLOSED pgs=128 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:31:17.503 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:17.501+0000 7f957f59e700 1 --2- 172.21.15.66:0/4269881669 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f95800f51a0 0x7f9580109170 unknown :-1 s=CLOSED pgs=326 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:31:17.503 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:17.501+0000 7f957f59e700 1 -- 172.21.15.66:0/4269881669 >> 172.21.15.66:0/4269881669 conn(0x7f95800f09a0 msgr2=0x7f95800f1460 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:31:17.503 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:17.501+0000 7f957f59e700 1 -- 172.21.15.66:0/4269881669 shutdown_connections 2023-11-17T05:31:17.503 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:17.501+0000 7f957f59e700 1 -- 172.21.15.66:0/4269881669 wait complete. 2023-11-17T05:31:17.503 INFO:teuthology.orchestra.run.smithi066.stderr:dumped monmap epoch 1 2023-11-17T05:31:17.988 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:31:17 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:31:17.500345+0000 mon.smithi040 (mon.0) 273 : audit [DBG] from='client.? 172.21.15.66:0/4269881669' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-11-17T05:31:17.989 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:31:17 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:31:17.513618+0000 mgr.smithi040.ogcprl (mgr.14164) 314 : cluster [DBG] pgmap v298: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:31:18.886 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2023-11-17T05:31:18.886 DEBUG:teuthology.orchestra.run.smithi066:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.4 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ed6470e2-8508-11ee-8dbe-212e2dc638e7 -- ceph mon dump -f json 2023-11-17T05:31:20.044 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:20.038+0000 7fbf4e052700 1 -- 172.21.15.66:0/3176274680 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fbf480f51a0 msgr2=0x7fbf480f55c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:31:20.044 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:20.038+0000 7fbf4e052700 1 --2- 172.21.15.66:0/3176274680 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fbf480f51a0 0x7fbf480f55c0 secure :-1 s=READY pgs=327 cs=0 l=1 rev1=1 rx=0x7fbf3c009fb0 tx=0x7fbf3c009b30).stop 2023-11-17T05:31:20.044 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:20.038+0000 7fbf4e052700 1 -- 172.21.15.66:0/3176274680 shutdown_connections 2023-11-17T05:31:20.044 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:20.038+0000 7fbf4e052700 1 --2- 172.21.15.66:0/3176274680 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fbf480f51a0 0x7fbf480f55c0 unknown :-1 s=CLOSED pgs=327 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:31:20.044 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:20.038+0000 7fbf4e052700 1 -- 172.21.15.66:0/3176274680 >> 172.21.15.66:0/3176274680 conn(0x7fbf480f09a0 msgr2=0x7fbf480f2dc0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:31:20.045 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:20.038+0000 7fbf4e052700 1 -- 172.21.15.66:0/3176274680 shutdown_connections 2023-11-17T05:31:20.045 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:20.038+0000 7fbf4e052700 1 -- 172.21.15.66:0/3176274680 wait complete. 2023-11-17T05:31:20.045 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:20.038+0000 7fbf4e052700 1 Processor -- start 2023-11-17T05:31:20.045 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:20.038+0000 7fbf4e052700 1 -- start start 2023-11-17T05:31:20.045 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:20.039+0000 7fbf4e052700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fbf480f51a0 0x7fbf481091d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:31:20.046 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:20.039+0000 7fbf4e052700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fbf48060220 con 0x7fbf480f51a0 2023-11-17T05:31:20.046 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:20.039+0000 7fbf4d050700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fbf480f51a0 0x7fbf481091d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:31:20.046 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:20.039+0000 7fbf4d050700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fbf480f51a0 0x7fbf481091d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.66:43562/0 (socket says 172.21.15.66:43562) 2023-11-17T05:31:20.046 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:20.039+0000 7fbf4d050700 1 -- 172.21.15.66:0/1871834595 learned_addr learned my addr 172.21.15.66:0/1871834595 (peer_addr_for_me v2:172.21.15.66:0/0) 2023-11-17T05:31:20.046 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:20.039+0000 7fbf4d050700 1 -- 172.21.15.66:0/1871834595 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fbf3c005040 con 0x7fbf480f51a0 2023-11-17T05:31:20.046 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:20.039+0000 7fbf4d050700 1 --2- 172.21.15.66:0/1871834595 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fbf480f51a0 0x7fbf481091d0 secure :-1 s=READY pgs=328 cs=0 l=1 rev1=1 rx=0x7fbf3c013510 tx=0x7fbf3c00b040).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:31:20.046 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:20.039+0000 7fbf39ffb700 1 -- 172.21.15.66:0/1871834595 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7fbf3c009320 con 0x7fbf480f51a0 2023-11-17T05:31:20.047 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:20.039+0000 7fbf4e052700 1 -- 172.21.15.66:0/1871834595 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fbf48060420 con 0x7fbf480f51a0 2023-11-17T05:31:20.047 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:20.039+0000 7fbf39ffb700 1 -- 172.21.15.66:0/1871834595 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(25 keys) v1 ==== 1074+0+0 (secure 0 0 0) 0x7fbf3c013f00 con 0x7fbf480f51a0 2023-11-17T05:31:20.047 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:20.039+0000 7fbf39ffb700 1 -- 172.21.15.66:0/1871834595 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7fbf3c014dc0 con 0x7fbf480f51a0 2023-11-17T05:31:20.047 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:20.040+0000 7fbf4e052700 1 -- 172.21.15.66:0/1871834595 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fbf48109880 con 0x7fbf480f51a0 2023-11-17T05:31:20.048 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:20.041+0000 7fbf4e052700 1 -- 172.21.15.66:0/1871834595 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fbf48059100 con 0x7fbf480f51a0 2023-11-17T05:31:20.048 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:20.041+0000 7fbf39ffb700 1 -- 172.21.15.66:0/1871834595 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 17) v1 ==== 36269+0+0 (secure 0 0 0) 0x7fbf3c0095f0 con 0x7fbf480f51a0 2023-11-17T05:31:20.048 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:20.041+0000 7fbf39ffb700 1 --2- 172.21.15.66:0/1871834595 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fbf3402ff60 0x7fbf34032420 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:31:20.048 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:20.041+0000 7fbf39ffb700 1 -- 172.21.15.66:0/1871834595 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(5..5 src has 1..5) v4 ==== 1270+0+0 (secure 0 0 0) 0x7fbf3c041040 con 0x7fbf480f51a0 2023-11-17T05:31:20.048 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:20.041+0000 7fbf4c84f700 1 --2- 172.21.15.66:0/1871834595 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fbf3402ff60 0x7fbf34032420 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:31:20.049 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:20.041+0000 7fbf4c84f700 1 --2- 172.21.15.66:0/1871834595 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fbf3402ff60 0x7fbf34032420 secure :-1 s=READY pgs=129 cs=0 l=1 rev1=1 rx=0x7fbf440096a0 tx=0x7fbf44006b40).ready entity=mgr.14164 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:31:20.049 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:20.046+0000 7fbf39ffb700 1 -- 172.21.15.66:0/1871834595 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+152212 (secure 0 0 0) 0x7fbf3c014420 con 0x7fbf480f51a0 2023-11-17T05:31:20.268 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:20.266+0000 7fbf4e052700 1 -- 172.21.15.66:0/1871834595 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "mon dump", "format": "json"} v 0) v1 -- 0x7fbf48045ad0 con 0x7fbf480f51a0 2023-11-17T05:31:20.272 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:20.268+0000 7fbf39ffb700 1 -- 172.21.15.66:0/1871834595 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mon_command_ack([{"prefix": "mon dump", "format": "json"}]=0 dumped monmap epoch 1 v1) v1 ==== 95+0+690 (secure 0 0 0) 0x7fbf3c010230 con 0x7fbf480f51a0 2023-11-17T05:31:20.277 INFO:teuthology.orchestra.run.smithi066.stdout: 2023-11-17T05:31:20.277 INFO:teuthology.orchestra.run.smithi066.stdout:{"epoch":1,"fsid":"ed6470e2-8508-11ee-8dbe-212e2dc638e7","modified":"2023-11-17T05:20:13.934038Z","created":"2023-11-17T05:20:13.934038Z","min_mon_release":16,"min_mon_release_name":"pacific","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging"],"optional":[]},"mons":[{"rank":0,"name":"smithi040","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:3300","nonce":0},{"type":"v1","addr":"172.21.15.40:6789","nonce":0}]},"addr":"172.21.15.40:6789/0","public_addr":"172.21.15.40:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2023-11-17T05:31:20.280 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:20.268+0000 7fbf4e052700 1 -- 172.21.15.66:0/1871834595 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fbf3402ff60 msgr2=0x7fbf34032420 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:31:20.281 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:20.268+0000 7fbf4e052700 1 --2- 172.21.15.66:0/1871834595 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fbf3402ff60 0x7fbf34032420 secure :-1 s=READY pgs=129 cs=0 l=1 rev1=1 rx=0x7fbf440096a0 tx=0x7fbf44006b40).stop 2023-11-17T05:31:20.281 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:20.268+0000 7fbf4e052700 1 -- 172.21.15.66:0/1871834595 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fbf480f51a0 msgr2=0x7fbf481091d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:31:20.282 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:20.268+0000 7fbf4e052700 1 --2- 172.21.15.66:0/1871834595 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fbf480f51a0 0x7fbf481091d0 secure :-1 s=READY pgs=328 cs=0 l=1 rev1=1 rx=0x7fbf3c013510 tx=0x7fbf3c00b040).stop 2023-11-17T05:31:20.282 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:20.269+0000 7fbf4e052700 1 -- 172.21.15.66:0/1871834595 shutdown_connections 2023-11-17T05:31:20.283 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:20.269+0000 7fbf4e052700 1 --2- 172.21.15.66:0/1871834595 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fbf3402ff60 0x7fbf34032420 unknown :-1 s=CLOSED pgs=129 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:31:20.283 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:20.269+0000 7fbf4e052700 1 --2- 172.21.15.66:0/1871834595 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fbf480f51a0 0x7fbf481091d0 unknown :-1 s=CLOSED pgs=328 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:31:20.283 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:20.269+0000 7fbf4e052700 1 -- 172.21.15.66:0/1871834595 >> 172.21.15.66:0/1871834595 conn(0x7fbf480f09a0 msgr2=0x7fbf480f1460 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:31:20.284 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:20.269+0000 7fbf4e052700 1 -- 172.21.15.66:0/1871834595 shutdown_connections 2023-11-17T05:31:20.284 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:20.269+0000 7fbf4e052700 1 -- 172.21.15.66:0/1871834595 wait complete. 2023-11-17T05:31:20.285 INFO:teuthology.orchestra.run.smithi066.stderr:dumped monmap epoch 1 2023-11-17T05:31:20.989 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:31:20 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:31:19.513940+0000 mgr.smithi040.ogcprl (mgr.14164) 315 : cluster [DBG] pgmap v299: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:31:20.989 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:31:20 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:31:20.268645+0000 mon.smithi040 (mon.0) 274 : audit [DBG] from='client.? 172.21.15.66:0/1871834595' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-11-17T05:31:21.668 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2023-11-17T05:31:21.669 DEBUG:teuthology.orchestra.run.smithi066:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.4 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ed6470e2-8508-11ee-8dbe-212e2dc638e7 -- ceph mon dump -f json 2023-11-17T05:31:21.988 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:31:21 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:31:21.514306+0000 mgr.smithi040.ogcprl (mgr.14164) 316 : cluster [DBG] pgmap v300: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:31:22.828 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:22.826+0000 7f50b11ac700 1 -- 172.21.15.66:0/2792476008 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f50ac0f5410 msgr2=0x7f50ac0f5830 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:31:22.828 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:22.826+0000 7f50b11ac700 1 --2- 172.21.15.66:0/2792476008 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f50ac0f5410 0x7f50ac0f5830 secure :-1 s=READY pgs=329 cs=0 l=1 rev1=1 rx=0x7f509c009fb0 tx=0x7f509c009b30).stop 2023-11-17T05:31:22.828 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:22.826+0000 7f50b11ac700 1 -- 172.21.15.66:0/2792476008 shutdown_connections 2023-11-17T05:31:22.828 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:22.826+0000 7f50b11ac700 1 --2- 172.21.15.66:0/2792476008 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f50ac0f5410 0x7f50ac0f5830 unknown :-1 s=CLOSED pgs=329 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:31:22.828 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:22.826+0000 7f50b11ac700 1 -- 172.21.15.66:0/2792476008 >> 172.21.15.66:0/2792476008 conn(0x7f50ac0f0b70 msgr2=0x7f50ac0f2f90 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:31:22.829 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:22.826+0000 7f50b11ac700 1 -- 172.21.15.66:0/2792476008 shutdown_connections 2023-11-17T05:31:22.829 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:22.826+0000 7f50b11ac700 1 -- 172.21.15.66:0/2792476008 wait complete. 2023-11-17T05:31:22.829 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:22.827+0000 7f50b11ac700 1 Processor -- start 2023-11-17T05:31:22.829 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:22.827+0000 7f50b11ac700 1 -- start start 2023-11-17T05:31:22.829 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:22.827+0000 7f50b11ac700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f50ac0f5410 0x7f50ac0fd3a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:31:22.829 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:22.827+0000 7f50b11ac700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f50ac0fb820 con 0x7f50ac0f5410 2023-11-17T05:31:22.831 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:22.827+0000 7f50aad9d700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f50ac0f5410 0x7f50ac0fd3a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:31:22.831 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:22.827+0000 7f50aad9d700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f50ac0f5410 0x7f50ac0fd3a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.66:43466/0 (socket says 172.21.15.66:43466) 2023-11-17T05:31:22.831 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:22.827+0000 7f50aad9d700 1 -- 172.21.15.66:0/886181357 learned_addr learned my addr 172.21.15.66:0/886181357 (peer_addr_for_me v2:172.21.15.66:0/0) 2023-11-17T05:31:22.831 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:22.828+0000 7f50aad9d700 1 -- 172.21.15.66:0/886181357 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f509c005040 con 0x7f50ac0f5410 2023-11-17T05:31:22.831 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:22.828+0000 7f50aad9d700 1 --2- 172.21.15.66:0/886181357 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f50ac0f5410 0x7f50ac0fd3a0 secure :-1 s=READY pgs=330 cs=0 l=1 rev1=1 rx=0x7f509c013510 tx=0x7f509c00b040).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:31:22.831 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:22.828+0000 7f50a37fe700 1 -- 172.21.15.66:0/886181357 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f509c009320 con 0x7f50ac0f5410 2023-11-17T05:31:22.831 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:22.828+0000 7f50b11ac700 1 -- 172.21.15.66:0/886181357 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f50ac0fb9c0 con 0x7f50ac0f5410 2023-11-17T05:31:22.831 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:22.828+0000 7f50a37fe700 1 -- 172.21.15.66:0/886181357 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(25 keys) v1 ==== 1074+0+0 (secure 0 0 0) 0x7f509c013f00 con 0x7f50ac0f5410 2023-11-17T05:31:22.831 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:22.828+0000 7f50a37fe700 1 -- 172.21.15.66:0/886181357 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f509c014d90 con 0x7f50ac0f5410 2023-11-17T05:31:22.831 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:22.828+0000 7f50b11ac700 1 -- 172.21.15.66:0/886181357 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f50ac0fbe50 con 0x7f50ac0f5410 2023-11-17T05:31:22.831 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:22.829+0000 7f50a37fe700 1 -- 172.21.15.66:0/886181357 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 17) v1 ==== 36269+0+0 (secure 0 0 0) 0x7f509c0095f0 con 0x7f50ac0f5410 2023-11-17T05:31:22.831 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:22.829+0000 7f50b11ac700 1 -- 172.21.15.66:0/886181357 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f50ac059100 con 0x7f50ac0f5410 2023-11-17T05:31:22.832 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:22.829+0000 7f50a37fe700 1 --2- 172.21.15.66:0/886181357 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f509802ff60 0x7f5098032420 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:31:22.832 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:22.829+0000 7f50a37fe700 1 -- 172.21.15.66:0/886181357 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(5..5 src has 1..5) v4 ==== 1270+0+0 (secure 0 0 0) 0x7f509c040fc0 con 0x7f50ac0f5410 2023-11-17T05:31:22.843 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:22.829+0000 7f50aa59c700 1 --2- 172.21.15.66:0/886181357 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f509802ff60 0x7f5098032420 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:31:22.844 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:22.830+0000 7f50aa59c700 1 --2- 172.21.15.66:0/886181357 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f509802ff60 0x7f5098032420 secure :-1 s=READY pgs=130 cs=0 l=1 rev1=1 rx=0x7f50940096a0 tx=0x7f5094006b40).ready entity=mgr.14164 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:31:22.844 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:22.834+0000 7f50a37fe700 1 -- 172.21.15.66:0/886181357 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+152212 (secure 0 0 0) 0x7f509c014420 con 0x7f50ac0f5410 2023-11-17T05:31:23.053 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:23.051+0000 7f50b11ac700 1 -- 172.21.15.66:0/886181357 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "mon dump", "format": "json"} v 0) v1 -- 0x7f50ac045ad0 con 0x7f50ac0f5410 2023-11-17T05:31:23.056 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:23.052+0000 7f50a37fe700 1 -- 172.21.15.66:0/886181357 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mon_command_ack([{"prefix": "mon dump", "format": "json"}]=0 dumped monmap epoch 1 v1) v1 ==== 95+0+690 (secure 0 0 0) 0x7f509c010230 con 0x7f50ac0f5410 2023-11-17T05:31:23.059 INFO:teuthology.orchestra.run.smithi066.stdout: 2023-11-17T05:31:23.059 INFO:teuthology.orchestra.run.smithi066.stdout:{"epoch":1,"fsid":"ed6470e2-8508-11ee-8dbe-212e2dc638e7","modified":"2023-11-17T05:20:13.934038Z","created":"2023-11-17T05:20:13.934038Z","min_mon_release":16,"min_mon_release_name":"pacific","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging"],"optional":[]},"mons":[{"rank":0,"name":"smithi040","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:3300","nonce":0},{"type":"v1","addr":"172.21.15.40:6789","nonce":0}]},"addr":"172.21.15.40:6789/0","public_addr":"172.21.15.40:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2023-11-17T05:31:23.061 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:23.054+0000 7f50b11ac700 1 -- 172.21.15.66:0/886181357 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f509802ff60 msgr2=0x7f5098032420 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:31:23.062 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:23.054+0000 7f50b11ac700 1 --2- 172.21.15.66:0/886181357 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f509802ff60 0x7f5098032420 secure :-1 s=READY pgs=130 cs=0 l=1 rev1=1 rx=0x7f50940096a0 tx=0x7f5094006b40).stop 2023-11-17T05:31:23.062 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:23.054+0000 7f50b11ac700 1 -- 172.21.15.66:0/886181357 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f50ac0f5410 msgr2=0x7f50ac0fd3a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:31:23.063 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:23.054+0000 7f50b11ac700 1 --2- 172.21.15.66:0/886181357 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f50ac0f5410 0x7f50ac0fd3a0 secure :-1 s=READY pgs=330 cs=0 l=1 rev1=1 rx=0x7f509c013510 tx=0x7f509c00b040).stop 2023-11-17T05:31:23.063 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:23.054+0000 7f50b11ac700 1 -- 172.21.15.66:0/886181357 shutdown_connections 2023-11-17T05:31:23.064 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:23.054+0000 7f50b11ac700 1 --2- 172.21.15.66:0/886181357 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f509802ff60 0x7f5098032420 unknown :-1 s=CLOSED pgs=130 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:31:23.064 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:23.054+0000 7f50b11ac700 1 --2- 172.21.15.66:0/886181357 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f50ac0f5410 0x7f50ac0fd3a0 unknown :-1 s=CLOSED pgs=330 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:31:23.064 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:23.054+0000 7f50b11ac700 1 -- 172.21.15.66:0/886181357 >> 172.21.15.66:0/886181357 conn(0x7f50ac0f0b70 msgr2=0x7f50ac05fe60 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:31:23.064 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:23.054+0000 7f50b11ac700 1 -- 172.21.15.66:0/886181357 shutdown_connections 2023-11-17T05:31:23.064 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:23.054+0000 7f50b11ac700 1 -- 172.21.15.66:0/886181357 wait complete. 2023-11-17T05:31:23.064 INFO:teuthology.orchestra.run.smithi066.stderr:dumped monmap epoch 1 2023-11-17T05:31:23.489 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:31:23 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:31:23.053707+0000 mon.smithi040 (mon.0) 275 : audit [DBG] from='client.? 172.21.15.66:0/886181357' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-11-17T05:31:24.454 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2023-11-17T05:31:24.454 DEBUG:teuthology.orchestra.run.smithi066:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.4 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ed6470e2-8508-11ee-8dbe-212e2dc638e7 -- ceph mon dump -f json 2023-11-17T05:31:24.488 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:31:24 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:31:23.514755+0000 mgr.smithi040.ogcprl (mgr.14164) 317 : cluster [DBG] pgmap v301: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:31:25.605 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:25.603+0000 7f03bcc38700 1 -- 172.21.15.66:0/2506923943 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f03b80f6510 msgr2=0x7f03b80f6930 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:31:25.605 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:25.603+0000 7f03bcc38700 1 --2- 172.21.15.66:0/2506923943 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f03b80f6510 0x7f03b80f6930 secure :-1 s=READY pgs=331 cs=0 l=1 rev1=1 rx=0x7f03a8009fb0 tx=0x7f03a8009b30).stop 2023-11-17T05:31:25.605 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:25.603+0000 7f03bcc38700 1 -- 172.21.15.66:0/2506923943 shutdown_connections 2023-11-17T05:31:25.605 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:25.603+0000 7f03bcc38700 1 --2- 172.21.15.66:0/2506923943 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f03b80f6510 0x7f03b80f6930 unknown :-1 s=CLOSED pgs=331 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:31:25.606 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:25.603+0000 7f03bcc38700 1 -- 172.21.15.66:0/2506923943 >> 172.21.15.66:0/2506923943 conn(0x7f03b80f1c70 msgr2=0x7f03b80f4090 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:31:25.606 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:25.603+0000 7f03bcc38700 1 -- 172.21.15.66:0/2506923943 shutdown_connections 2023-11-17T05:31:25.606 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:25.603+0000 7f03bcc38700 1 -- 172.21.15.66:0/2506923943 wait complete. 2023-11-17T05:31:25.606 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:25.604+0000 7f03bcc38700 1 Processor -- start 2023-11-17T05:31:25.606 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:25.604+0000 7f03bcc38700 1 -- start start 2023-11-17T05:31:25.606 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:25.604+0000 7f03bcc38700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f03b80f6510 0x7f03b806b9a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:31:25.606 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:25.604+0000 7f03bcc38700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f03b8069e20 con 0x7f03b80f6510 2023-11-17T05:31:25.608 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:25.604+0000 7f03b659c700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f03b80f6510 0x7f03b806b9a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:31:25.608 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:25.604+0000 7f03b659c700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f03b80f6510 0x7f03b806b9a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.66:43482/0 (socket says 172.21.15.66:43482) 2023-11-17T05:31:25.608 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:25.604+0000 7f03b659c700 1 -- 172.21.15.66:0/1531988091 learned_addr learned my addr 172.21.15.66:0/1531988091 (peer_addr_for_me v2:172.21.15.66:0/0) 2023-11-17T05:31:25.608 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:25.605+0000 7f03b659c700 1 -- 172.21.15.66:0/1531988091 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f03a8005040 con 0x7f03b80f6510 2023-11-17T05:31:25.608 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:25.605+0000 7f03b659c700 1 --2- 172.21.15.66:0/1531988091 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f03b80f6510 0x7f03b806b9a0 secure :-1 s=READY pgs=332 cs=0 l=1 rev1=1 rx=0x7f03a8013510 tx=0x7f03a800b040).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:31:25.608 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:25.605+0000 7f03a6ffd700 1 -- 172.21.15.66:0/1531988091 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f03a8009320 con 0x7f03b80f6510 2023-11-17T05:31:25.608 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:25.605+0000 7f03a6ffd700 1 -- 172.21.15.66:0/1531988091 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(25 keys) v1 ==== 1074+0+0 (secure 0 0 0) 0x7f03a8013f00 con 0x7f03b80f6510 2023-11-17T05:31:25.608 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:25.605+0000 7f03bcc38700 1 -- 172.21.15.66:0/1531988091 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f03b8069fc0 con 0x7f03b80f6510 2023-11-17T05:31:25.608 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:25.605+0000 7f03a6ffd700 1 -- 172.21.15.66:0/1531988091 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f03a8014d90 con 0x7f03b80f6510 2023-11-17T05:31:25.608 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:25.605+0000 7f03bcc38700 1 -- 172.21.15.66:0/1531988091 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f03b806a450 con 0x7f03b80f6510 2023-11-17T05:31:25.609 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:25.606+0000 7f03bcc38700 1 -- 172.21.15.66:0/1531988091 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f03b8059100 con 0x7f03b80f6510 2023-11-17T05:31:25.609 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:25.606+0000 7f03a6ffd700 1 -- 172.21.15.66:0/1531988091 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 17) v1 ==== 36269+0+0 (secure 0 0 0) 0x7f03a80095f0 con 0x7f03b80f6510 2023-11-17T05:31:25.609 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:25.606+0000 7f03a6ffd700 1 --2- 172.21.15.66:0/1531988091 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f03a002ff60 0x7f03a0032420 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:31:25.609 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:25.606+0000 7f03a6ffd700 1 -- 172.21.15.66:0/1531988091 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(5..5 src has 1..5) v4 ==== 1270+0+0 (secure 0 0 0) 0x7f03a80410f0 con 0x7f03b80f6510 2023-11-17T05:31:25.609 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:25.606+0000 7f03b5d9b700 1 --2- 172.21.15.66:0/1531988091 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f03a002ff60 0x7f03a0032420 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:31:25.609 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:25.607+0000 7f03b5d9b700 1 --2- 172.21.15.66:0/1531988091 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f03a002ff60 0x7f03a0032420 secure :-1 s=READY pgs=131 cs=0 l=1 rev1=1 rx=0x7f03ac0096a0 tx=0x7f03ac006b40).ready entity=mgr.14164 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:31:25.612 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:25.611+0000 7f03a6ffd700 1 -- 172.21.15.66:0/1531988091 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+152212 (secure 0 0 0) 0x7f03a8014420 con 0x7f03b80f6510 2023-11-17T05:31:25.829 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:25.828+0000 7f03bcc38700 1 -- 172.21.15.66:0/1531988091 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "mon dump", "format": "json"} v 0) v1 -- 0x7f03b80fbf60 con 0x7f03b80f6510 2023-11-17T05:31:25.830 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:25.829+0000 7f03a6ffd700 1 -- 172.21.15.66:0/1531988091 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mon_command_ack([{"prefix": "mon dump", "format": "json"}]=0 dumped monmap epoch 1 v1) v1 ==== 95+0+690 (secure 0 0 0) 0x7f03a8010230 con 0x7f03b80f6510 2023-11-17T05:31:25.831 INFO:teuthology.orchestra.run.smithi066.stdout: 2023-11-17T05:31:25.831 INFO:teuthology.orchestra.run.smithi066.stdout:{"epoch":1,"fsid":"ed6470e2-8508-11ee-8dbe-212e2dc638e7","modified":"2023-11-17T05:20:13.934038Z","created":"2023-11-17T05:20:13.934038Z","min_mon_release":16,"min_mon_release_name":"pacific","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging"],"optional":[]},"mons":[{"rank":0,"name":"smithi040","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:3300","nonce":0},{"type":"v1","addr":"172.21.15.40:6789","nonce":0}]},"addr":"172.21.15.40:6789/0","public_addr":"172.21.15.40:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2023-11-17T05:31:25.832 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:25.830+0000 7f03bcc38700 1 -- 172.21.15.66:0/1531988091 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f03a002ff60 msgr2=0x7f03a0032420 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:31:25.832 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:25.830+0000 7f03bcc38700 1 --2- 172.21.15.66:0/1531988091 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f03a002ff60 0x7f03a0032420 secure :-1 s=READY pgs=131 cs=0 l=1 rev1=1 rx=0x7f03ac0096a0 tx=0x7f03ac006b40).stop 2023-11-17T05:31:25.832 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:25.831+0000 7f03bcc38700 1 -- 172.21.15.66:0/1531988091 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f03b80f6510 msgr2=0x7f03b806b9a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:31:25.832 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:25.831+0000 7f03bcc38700 1 --2- 172.21.15.66:0/1531988091 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f03b80f6510 0x7f03b806b9a0 secure :-1 s=READY pgs=332 cs=0 l=1 rev1=1 rx=0x7f03a8013510 tx=0x7f03a800b040).stop 2023-11-17T05:31:25.832 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:25.831+0000 7f03bcc38700 1 -- 172.21.15.66:0/1531988091 shutdown_connections 2023-11-17T05:31:25.832 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:25.831+0000 7f03bcc38700 1 --2- 172.21.15.66:0/1531988091 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f03a002ff60 0x7f03a0032420 unknown :-1 s=CLOSED pgs=131 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:31:25.832 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:25.831+0000 7f03bcc38700 1 --2- 172.21.15.66:0/1531988091 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f03b80f6510 0x7f03b806b9a0 unknown :-1 s=CLOSED pgs=332 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:31:25.832 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:25.831+0000 7f03bcc38700 1 -- 172.21.15.66:0/1531988091 >> 172.21.15.66:0/1531988091 conn(0x7f03b80f1c70 msgr2=0x7f03b80f2760 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:31:25.832 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:25.831+0000 7f03bcc38700 1 -- 172.21.15.66:0/1531988091 shutdown_connections 2023-11-17T05:31:25.833 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:25.831+0000 7f03bcc38700 1 -- 172.21.15.66:0/1531988091 wait complete. 2023-11-17T05:31:25.833 INFO:teuthology.orchestra.run.smithi066.stderr:dumped monmap epoch 1 2023-11-17T05:31:25.988 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:31:25 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:31:25.515163+0000 mgr.smithi040.ogcprl (mgr.14164) 318 : cluster [DBG] pgmap v302: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:31:26.991 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:31:26 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:31:25.830431+0000 mon.smithi040 (mon.0) 276 : audit [DBG] from='client.? 172.21.15.66:0/1531988091' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-11-17T05:31:27.258 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2023-11-17T05:31:27.259 DEBUG:teuthology.orchestra.run.smithi066:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.4 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ed6470e2-8508-11ee-8dbe-212e2dc638e7 -- ceph mon dump -f json 2023-11-17T05:31:27.988 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:31:27 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:31:27.515494+0000 mgr.smithi040.ogcprl (mgr.14164) 319 : cluster [DBG] pgmap v303: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:31:28.428 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:28.425+0000 7fbb8cca6700 1 -- 172.21.15.66:0/3224270818 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fbb880f5330 msgr2=0x7fbb880f5750 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:31:28.428 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:28.425+0000 7fbb8cca6700 1 --2- 172.21.15.66:0/3224270818 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fbb880f5330 0x7fbb880f5750 secure :-1 s=READY pgs=333 cs=0 l=1 rev1=1 rx=0x7fbb78009fb0 tx=0x7fbb78009b30).stop 2023-11-17T05:31:28.428 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:28.425+0000 7fbb8cca6700 1 -- 172.21.15.66:0/3224270818 shutdown_connections 2023-11-17T05:31:28.428 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:28.425+0000 7fbb8cca6700 1 --2- 172.21.15.66:0/3224270818 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fbb880f5330 0x7fbb880f5750 unknown :-1 s=CLOSED pgs=333 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:31:28.428 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:28.425+0000 7fbb8cca6700 1 -- 172.21.15.66:0/3224270818 >> 172.21.15.66:0/3224270818 conn(0x7fbb880f0a90 msgr2=0x7fbb880f2eb0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:31:28.428 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:28.425+0000 7fbb8cca6700 1 -- 172.21.15.66:0/3224270818 shutdown_connections 2023-11-17T05:31:28.428 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:28.425+0000 7fbb8cca6700 1 -- 172.21.15.66:0/3224270818 wait complete. 2023-11-17T05:31:28.428 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:28.425+0000 7fbb8cca6700 1 Processor -- start 2023-11-17T05:31:28.428 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:28.425+0000 7fbb8cca6700 1 -- start start 2023-11-17T05:31:28.429 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:28.426+0000 7fbb8cca6700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fbb880f5330 0x7fbb881096f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:31:28.429 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:28.426+0000 7fbb8cca6700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fbb88060220 con 0x7fbb880f5330 2023-11-17T05:31:28.429 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:28.426+0000 7fbb8659c700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fbb880f5330 0x7fbb881096f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:31:28.429 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:28.426+0000 7fbb8659c700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fbb880f5330 0x7fbb881096f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.66:43498/0 (socket says 172.21.15.66:43498) 2023-11-17T05:31:28.429 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:28.426+0000 7fbb8659c700 1 -- 172.21.15.66:0/1648671935 learned_addr learned my addr 172.21.15.66:0/1648671935 (peer_addr_for_me v2:172.21.15.66:0/0) 2023-11-17T05:31:28.429 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:28.426+0000 7fbb8659c700 1 -- 172.21.15.66:0/1648671935 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fbb78005040 con 0x7fbb880f5330 2023-11-17T05:31:28.429 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:28.426+0000 7fbb8659c700 1 --2- 172.21.15.66:0/1648671935 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fbb880f5330 0x7fbb881096f0 secure :-1 s=READY pgs=334 cs=0 l=1 rev1=1 rx=0x7fbb78013510 tx=0x7fbb7800b040).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:31:28.429 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:28.426+0000 7fbb7effd700 1 -- 172.21.15.66:0/1648671935 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7fbb78009320 con 0x7fbb880f5330 2023-11-17T05:31:28.429 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:28.426+0000 7fbb7effd700 1 -- 172.21.15.66:0/1648671935 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(25 keys) v1 ==== 1074+0+0 (secure 0 0 0) 0x7fbb78013f00 con 0x7fbb880f5330 2023-11-17T05:31:28.429 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:28.426+0000 7fbb8cca6700 1 -- 172.21.15.66:0/1648671935 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fbb88060420 con 0x7fbb880f5330 2023-11-17T05:31:28.429 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:28.426+0000 7fbb7effd700 1 -- 172.21.15.66:0/1648671935 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7fbb78014d90 con 0x7fbb880f5330 2023-11-17T05:31:28.430 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:28.426+0000 7fbb8cca6700 1 -- 172.21.15.66:0/1648671935 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fbb88109da0 con 0x7fbb880f5330 2023-11-17T05:31:28.430 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:28.427+0000 7fbb7effd700 1 -- 172.21.15.66:0/1648671935 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 17) v1 ==== 36269+0+0 (secure 0 0 0) 0x7fbb780095f0 con 0x7fbb880f5330 2023-11-17T05:31:28.431 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:28.427+0000 7fbb8cca6700 1 -- 172.21.15.66:0/1648671935 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fbb88045ad0 con 0x7fbb880f5330 2023-11-17T05:31:28.431 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:28.427+0000 7fbb7effd700 1 --2- 172.21.15.66:0/1648671935 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fbb7402ff60 0x7fbb74032420 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:31:28.431 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:28.427+0000 7fbb7effd700 1 -- 172.21.15.66:0/1648671935 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(5..5 src has 1..5) v4 ==== 1270+0+0 (secure 0 0 0) 0x7fbb78041000 con 0x7fbb880f5330 2023-11-17T05:31:28.431 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:28.428+0000 7fbb85d9b700 1 --2- 172.21.15.66:0/1648671935 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fbb7402ff60 0x7fbb74032420 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:31:28.431 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:28.428+0000 7fbb85d9b700 1 --2- 172.21.15.66:0/1648671935 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fbb7402ff60 0x7fbb74032420 secure :-1 s=READY pgs=132 cs=0 l=1 rev1=1 rx=0x7fbb700096a0 tx=0x7fbb70006b40).ready entity=mgr.14164 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:31:28.433 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:28.432+0000 7fbb7effd700 1 -- 172.21.15.66:0/1648671935 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+152212 (secure 0 0 0) 0x7fbb78014420 con 0x7fbb880f5330 2023-11-17T05:31:28.650 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:28.649+0000 7fbb8cca6700 1 -- 172.21.15.66:0/1648671935 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "mon dump", "format": "json"} v 0) v1 -- 0x7fbb8805ea50 con 0x7fbb880f5330 2023-11-17T05:31:28.650 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:28.649+0000 7fbb7effd700 1 -- 172.21.15.66:0/1648671935 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mon_command_ack([{"prefix": "mon dump", "format": "json"}]=0 dumped monmap epoch 1 v1) v1 ==== 95+0+690 (secure 0 0 0) 0x7fbb7801d7f0 con 0x7fbb880f5330 2023-11-17T05:31:28.651 INFO:teuthology.orchestra.run.smithi066.stdout: 2023-11-17T05:31:28.651 INFO:teuthology.orchestra.run.smithi066.stdout:{"epoch":1,"fsid":"ed6470e2-8508-11ee-8dbe-212e2dc638e7","modified":"2023-11-17T05:20:13.934038Z","created":"2023-11-17T05:20:13.934038Z","min_mon_release":16,"min_mon_release_name":"pacific","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging"],"optional":[]},"mons":[{"rank":0,"name":"smithi040","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:3300","nonce":0},{"type":"v1","addr":"172.21.15.40:6789","nonce":0}]},"addr":"172.21.15.40:6789/0","public_addr":"172.21.15.40:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2023-11-17T05:31:28.652 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:28.650+0000 7fbb8cca6700 1 -- 172.21.15.66:0/1648671935 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fbb7402ff60 msgr2=0x7fbb74032420 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:31:28.652 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:28.650+0000 7fbb8cca6700 1 --2- 172.21.15.66:0/1648671935 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fbb7402ff60 0x7fbb74032420 secure :-1 s=READY pgs=132 cs=0 l=1 rev1=1 rx=0x7fbb700096a0 tx=0x7fbb70006b40).stop 2023-11-17T05:31:28.652 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:28.650+0000 7fbb8cca6700 1 -- 172.21.15.66:0/1648671935 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fbb880f5330 msgr2=0x7fbb881096f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:31:28.652 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:28.650+0000 7fbb8cca6700 1 --2- 172.21.15.66:0/1648671935 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fbb880f5330 0x7fbb881096f0 secure :-1 s=READY pgs=334 cs=0 l=1 rev1=1 rx=0x7fbb78013510 tx=0x7fbb7800b040).stop 2023-11-17T05:31:28.652 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:28.651+0000 7fbb8cca6700 1 -- 172.21.15.66:0/1648671935 shutdown_connections 2023-11-17T05:31:28.652 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:28.651+0000 7fbb8cca6700 1 --2- 172.21.15.66:0/1648671935 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fbb7402ff60 0x7fbb74032420 unknown :-1 s=CLOSED pgs=132 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:31:28.652 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:28.651+0000 7fbb8cca6700 1 --2- 172.21.15.66:0/1648671935 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fbb880f5330 0x7fbb881096f0 unknown :-1 s=CLOSED pgs=334 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:31:28.652 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:28.651+0000 7fbb8cca6700 1 -- 172.21.15.66:0/1648671935 >> 172.21.15.66:0/1648671935 conn(0x7fbb880f0a90 msgr2=0x7fbb880f1530 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:31:28.652 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:28.651+0000 7fbb8cca6700 1 -- 172.21.15.66:0/1648671935 shutdown_connections 2023-11-17T05:31:28.653 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:28.651+0000 7fbb8cca6700 1 -- 172.21.15.66:0/1648671935 wait complete. 2023-11-17T05:31:28.653 INFO:teuthology.orchestra.run.smithi066.stderr:dumped monmap epoch 1 2023-11-17T05:31:28.989 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:31:28 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:31:28.650836+0000 mon.smithi040 (mon.0) 277 : audit [DBG] from='client.? 172.21.15.66:0/1648671935' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-11-17T05:31:30.055 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2023-11-17T05:31:30.055 DEBUG:teuthology.orchestra.run.smithi066:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.4 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ed6470e2-8508-11ee-8dbe-212e2dc638e7 -- ceph mon dump -f json 2023-11-17T05:31:30.988 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:31:30 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:31:29.515760+0000 mgr.smithi040.ogcprl (mgr.14164) 320 : cluster [DBG] pgmap v304: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:31:31.277 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:31.275+0000 7ff90b59e700 1 -- 172.21.15.66:0/282361283 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7ff90c0f51a0 msgr2=0x7ff90c0f55c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:31:31.277 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:31.275+0000 7ff90b59e700 1 --2- 172.21.15.66:0/282361283 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7ff90c0f51a0 0x7ff90c0f55c0 secure :-1 s=READY pgs=335 cs=0 l=1 rev1=1 rx=0x7ff8fc009fb0 tx=0x7ff8fc009b30).stop 2023-11-17T05:31:31.277 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:31.275+0000 7ff90b59e700 1 -- 172.21.15.66:0/282361283 shutdown_connections 2023-11-17T05:31:31.277 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:31.275+0000 7ff90b59e700 1 --2- 172.21.15.66:0/282361283 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7ff90c0f51a0 0x7ff90c0f55c0 unknown :-1 s=CLOSED pgs=335 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:31:31.277 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:31.275+0000 7ff90b59e700 1 -- 172.21.15.66:0/282361283 >> 172.21.15.66:0/282361283 conn(0x7ff90c0f09a0 msgr2=0x7ff90c0f2dc0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:31:31.277 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:31.275+0000 7ff90b59e700 1 -- 172.21.15.66:0/282361283 shutdown_connections 2023-11-17T05:31:31.277 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:31.275+0000 7ff90b59e700 1 -- 172.21.15.66:0/282361283 wait complete. 2023-11-17T05:31:31.277 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:31.276+0000 7ff90b59e700 1 Processor -- start 2023-11-17T05:31:31.277 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:31.276+0000 7ff90b59e700 1 -- start start 2023-11-17T05:31:31.279 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:31.276+0000 7ff90b59e700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7ff90c0f51a0 0x7ff90c1053b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:31:31.279 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:31.276+0000 7ff90b59e700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7ff90c1058a0 con 0x7ff90c0f51a0 2023-11-17T05:31:31.279 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:31.276+0000 7ff90a59c700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7ff90c0f51a0 0x7ff90c1053b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:31:31.279 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:31.276+0000 7ff90a59c700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7ff90c0f51a0 0x7ff90c1053b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.66:43512/0 (socket says 172.21.15.66:43512) 2023-11-17T05:31:31.279 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:31.276+0000 7ff90a59c700 1 -- 172.21.15.66:0/1418551974 learned_addr learned my addr 172.21.15.66:0/1418551974 (peer_addr_for_me v2:172.21.15.66:0/0) 2023-11-17T05:31:31.279 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:31.277+0000 7ff90a59c700 1 -- 172.21.15.66:0/1418551974 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7ff8fc005040 con 0x7ff90c0f51a0 2023-11-17T05:31:31.279 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:31.277+0000 7ff90a59c700 1 --2- 172.21.15.66:0/1418551974 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7ff90c0f51a0 0x7ff90c1053b0 secure :-1 s=READY pgs=336 cs=0 l=1 rev1=1 rx=0x7ff8fc013510 tx=0x7ff8fc00b040).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:31:31.279 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:31.277+0000 7ff8faffd700 1 -- 172.21.15.66:0/1418551974 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7ff8fc009320 con 0x7ff90c0f51a0 2023-11-17T05:31:31.279 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:31.277+0000 7ff90b59e700 1 -- 172.21.15.66:0/1418551974 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7ff90c105aa0 con 0x7ff90c0f51a0 2023-11-17T05:31:31.279 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:31.277+0000 7ff8faffd700 1 -- 172.21.15.66:0/1418551974 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(25 keys) v1 ==== 1074+0+0 (secure 0 0 0) 0x7ff8fc013f00 con 0x7ff90c0f51a0 2023-11-17T05:31:31.279 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:31.277+0000 7ff8faffd700 1 -- 172.21.15.66:0/1418551974 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7ff8fc014dc0 con 0x7ff90c0f51a0 2023-11-17T05:31:31.280 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:31.277+0000 7ff90b59e700 1 -- 172.21.15.66:0/1418551974 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7ff90c105f30 con 0x7ff90c0f51a0 2023-11-17T05:31:31.280 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:31.278+0000 7ff90b59e700 1 -- 172.21.15.66:0/1418551974 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7ff90c045ad0 con 0x7ff90c0f51a0 2023-11-17T05:31:31.280 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:31.278+0000 7ff8faffd700 1 -- 172.21.15.66:0/1418551974 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 17) v1 ==== 36269+0+0 (secure 0 0 0) 0x7ff8fc0095f0 con 0x7ff90c0f51a0 2023-11-17T05:31:31.281 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:31.278+0000 7ff8faffd700 1 --2- 172.21.15.66:0/1418551974 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7ff8f402ff60 0x7ff8f4032420 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:31:31.281 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:31.279+0000 7ff8faffd700 1 -- 172.21.15.66:0/1418551974 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(5..5 src has 1..5) v4 ==== 1270+0+0 (secure 0 0 0) 0x7ff8fc041000 con 0x7ff90c0f51a0 2023-11-17T05:31:31.281 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:31.279+0000 7ff909d9b700 1 --2- 172.21.15.66:0/1418551974 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7ff8f402ff60 0x7ff8f4032420 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:31:31.281 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:31.279+0000 7ff909d9b700 1 --2- 172.21.15.66:0/1418551974 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7ff8f402ff60 0x7ff8f4032420 secure :-1 s=READY pgs=133 cs=0 l=1 rev1=1 rx=0x7ff9000096a0 tx=0x7ff900006b40).ready entity=mgr.14164 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:31:31.285 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:31.283+0000 7ff8faffd700 1 -- 172.21.15.66:0/1418551974 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+152212 (secure 0 0 0) 0x7ff8fc01c790 con 0x7ff90c0f51a0 2023-11-17T05:31:31.502 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:31.501+0000 7ff90b59e700 1 -- 172.21.15.66:0/1418551974 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "mon dump", "format": "json"} v 0) v1 -- 0x7ff90c106d80 con 0x7ff90c0f51a0 2023-11-17T05:31:31.503 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:31.501+0000 7ff8faffd700 1 -- 172.21.15.66:0/1418551974 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mon_command_ack([{"prefix": "mon dump", "format": "json"}]=0 dumped monmap epoch 1 v1) v1 ==== 95+0+690 (secure 0 0 0) 0x7ff8fc01c790 con 0x7ff90c0f51a0 2023-11-17T05:31:31.503 INFO:teuthology.orchestra.run.smithi066.stdout: 2023-11-17T05:31:31.503 INFO:teuthology.orchestra.run.smithi066.stdout:{"epoch":1,"fsid":"ed6470e2-8508-11ee-8dbe-212e2dc638e7","modified":"2023-11-17T05:20:13.934038Z","created":"2023-11-17T05:20:13.934038Z","min_mon_release":16,"min_mon_release_name":"pacific","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging"],"optional":[]},"mons":[{"rank":0,"name":"smithi040","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:3300","nonce":0},{"type":"v1","addr":"172.21.15.40:6789","nonce":0}]},"addr":"172.21.15.40:6789/0","public_addr":"172.21.15.40:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2023-11-17T05:31:31.517 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:31.503+0000 7ff90b59e700 1 -- 172.21.15.66:0/1418551974 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7ff8f402ff60 msgr2=0x7ff8f4032420 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:31:31.517 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:31.503+0000 7ff90b59e700 1 --2- 172.21.15.66:0/1418551974 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7ff8f402ff60 0x7ff8f4032420 secure :-1 s=READY pgs=133 cs=0 l=1 rev1=1 rx=0x7ff9000096a0 tx=0x7ff900006b40).stop 2023-11-17T05:31:31.518 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:31.503+0000 7ff90b59e700 1 -- 172.21.15.66:0/1418551974 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7ff90c0f51a0 msgr2=0x7ff90c1053b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:31:31.518 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:31.503+0000 7ff90b59e700 1 --2- 172.21.15.66:0/1418551974 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7ff90c0f51a0 0x7ff90c1053b0 secure :-1 s=READY pgs=336 cs=0 l=1 rev1=1 rx=0x7ff8fc013510 tx=0x7ff8fc00b040).stop 2023-11-17T05:31:31.518 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:31.503+0000 7ff90b59e700 1 -- 172.21.15.66:0/1418551974 shutdown_connections 2023-11-17T05:31:31.518 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:31.503+0000 7ff90b59e700 1 --2- 172.21.15.66:0/1418551974 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7ff8f402ff60 0x7ff8f4032420 unknown :-1 s=CLOSED pgs=133 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:31:31.518 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:31.503+0000 7ff90b59e700 1 --2- 172.21.15.66:0/1418551974 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7ff90c0f51a0 0x7ff90c1053b0 unknown :-1 s=CLOSED pgs=336 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:31:31.518 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:31.503+0000 7ff90b59e700 1 -- 172.21.15.66:0/1418551974 >> 172.21.15.66:0/1418551974 conn(0x7ff90c0f09a0 msgr2=0x7ff90c0f1460 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:31:31.518 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:31.503+0000 7ff90b59e700 1 -- 172.21.15.66:0/1418551974 shutdown_connections 2023-11-17T05:31:31.518 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:31.503+0000 7ff90b59e700 1 -- 172.21.15.66:0/1418551974 wait complete. 2023-11-17T05:31:31.518 INFO:teuthology.orchestra.run.smithi066.stderr:dumped monmap epoch 1 2023-11-17T05:31:31.988 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:31:31 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:31:31.503171+0000 mon.smithi040 (mon.0) 278 : audit [DBG] from='client.? 172.21.15.66:0/1418551974' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-11-17T05:31:31.989 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:31:31 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:31:31.515988+0000 mgr.smithi040.ogcprl (mgr.14164) 321 : cluster [DBG] pgmap v305: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:31:32.894 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2023-11-17T05:31:32.894 DEBUG:teuthology.orchestra.run.smithi066:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.4 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ed6470e2-8508-11ee-8dbe-212e2dc638e7 -- ceph mon dump -f json 2023-11-17T05:31:33.988 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:31:33 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:31:33.516292+0000 mgr.smithi040.ogcprl (mgr.14164) 322 : cluster [DBG] pgmap v306: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:31:34.056 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:34.052+0000 7fa750208700 1 -- 172.21.15.66:0/2658766688 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fa7480f5410 msgr2=0x7fa7480f5830 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:31:34.056 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:34.052+0000 7fa750208700 1 --2- 172.21.15.66:0/2658766688 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fa7480f5410 0x7fa7480f5830 secure :-1 s=READY pgs=337 cs=0 l=1 rev1=1 rx=0x7fa744009fb0 tx=0x7fa744009b30).stop 2023-11-17T05:31:34.056 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:34.052+0000 7fa750208700 1 -- 172.21.15.66:0/2658766688 shutdown_connections 2023-11-17T05:31:34.056 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:34.052+0000 7fa750208700 1 --2- 172.21.15.66:0/2658766688 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fa7480f5410 0x7fa7480f5830 unknown :-1 s=CLOSED pgs=337 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:31:34.056 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:34.052+0000 7fa750208700 1 -- 172.21.15.66:0/2658766688 >> 172.21.15.66:0/2658766688 conn(0x7fa7480f0b70 msgr2=0x7fa7480f2f90 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:31:34.056 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:34.053+0000 7fa750208700 1 -- 172.21.15.66:0/2658766688 shutdown_connections 2023-11-17T05:31:34.056 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:34.053+0000 7fa750208700 1 -- 172.21.15.66:0/2658766688 wait complete. 2023-11-17T05:31:34.056 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:34.053+0000 7fa750208700 1 Processor -- start 2023-11-17T05:31:34.056 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:34.053+0000 7fa750208700 1 -- start start 2023-11-17T05:31:34.056 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:34.053+0000 7fa750208700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fa7480f5410 0x7fa748109450 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:31:34.057 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:34.053+0000 7fa750208700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fa748060220 con 0x7fa7480f5410 2023-11-17T05:31:34.057 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:34.053+0000 7fa74dfa4700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fa7480f5410 0x7fa748109450 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:31:34.057 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:34.053+0000 7fa74dfa4700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fa7480f5410 0x7fa748109450 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.66:49886/0 (socket says 172.21.15.66:49886) 2023-11-17T05:31:34.057 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:34.053+0000 7fa74dfa4700 1 -- 172.21.15.66:0/2487202475 learned_addr learned my addr 172.21.15.66:0/2487202475 (peer_addr_for_me v2:172.21.15.66:0/0) 2023-11-17T05:31:34.057 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:34.054+0000 7fa74dfa4700 1 -- 172.21.15.66:0/2487202475 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fa744005040 con 0x7fa7480f5410 2023-11-17T05:31:34.057 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:34.054+0000 7fa74dfa4700 1 --2- 172.21.15.66:0/2487202475 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fa7480f5410 0x7fa748109450 secure :-1 s=READY pgs=338 cs=0 l=1 rev1=1 rx=0x7fa744013510 tx=0x7fa74400b040).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:31:34.057 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:34.054+0000 7fa73e7fc700 1 -- 172.21.15.66:0/2487202475 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7fa744009320 con 0x7fa7480f5410 2023-11-17T05:31:34.057 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:34.054+0000 7fa750208700 1 -- 172.21.15.66:0/2487202475 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fa748060420 con 0x7fa7480f5410 2023-11-17T05:31:34.057 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:34.054+0000 7fa73e7fc700 1 -- 172.21.15.66:0/2487202475 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(25 keys) v1 ==== 1074+0+0 (secure 0 0 0) 0x7fa744013f00 con 0x7fa7480f5410 2023-11-17T05:31:34.057 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:34.054+0000 7fa73e7fc700 1 -- 172.21.15.66:0/2487202475 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7fa744014d90 con 0x7fa7480f5410 2023-11-17T05:31:34.057 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:34.054+0000 7fa750208700 1 -- 172.21.15.66:0/2487202475 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fa748109b00 con 0x7fa7480f5410 2023-11-17T05:31:34.058 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:34.055+0000 7fa73e7fc700 1 -- 172.21.15.66:0/2487202475 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 17) v1 ==== 36269+0+0 (secure 0 0 0) 0x7fa7440095f0 con 0x7fa7480f5410 2023-11-17T05:31:34.058 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:34.055+0000 7fa750208700 1 -- 172.21.15.66:0/2487202475 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fa748059100 con 0x7fa7480f5410 2023-11-17T05:31:34.059 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:34.055+0000 7fa73e7fc700 1 --2- 172.21.15.66:0/2487202475 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fa73402ffb0 0x7fa734032470 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:31:34.059 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:34.055+0000 7fa73e7fc700 1 -- 172.21.15.66:0/2487202475 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(5..5 src has 1..5) v4 ==== 1270+0+0 (secure 0 0 0) 0x7fa744040fe0 con 0x7fa7480f5410 2023-11-17T05:31:34.059 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:34.055+0000 7fa74d7a3700 1 --2- 172.21.15.66:0/2487202475 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fa73402ffb0 0x7fa734032470 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:31:34.059 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:34.056+0000 7fa74d7a3700 1 --2- 172.21.15.66:0/2487202475 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fa73402ffb0 0x7fa734032470 secure :-1 s=READY pgs=134 cs=0 l=1 rev1=1 rx=0x7fa7380096a0 tx=0x7fa738006b40).ready entity=mgr.14164 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:31:34.064 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:34.060+0000 7fa73e7fc700 1 -- 172.21.15.66:0/2487202475 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+152212 (secure 0 0 0) 0x7fa74406d2b0 con 0x7fa7480f5410 2023-11-17T05:31:34.279 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:34.277+0000 7fa750208700 1 -- 172.21.15.66:0/2487202475 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "mon dump", "format": "json"} v 0) v1 -- 0x7fa748045ad0 con 0x7fa7480f5410 2023-11-17T05:31:34.279 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:34.278+0000 7fa73e7fc700 1 -- 172.21.15.66:0/2487202475 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mon_command_ack([{"prefix": "mon dump", "format": "json"}]=0 dumped monmap epoch 1 v1) v1 ==== 95+0+690 (secure 0 0 0) 0x7fa744014420 con 0x7fa7480f5410 2023-11-17T05:31:34.280 INFO:teuthology.orchestra.run.smithi066.stdout: 2023-11-17T05:31:34.280 INFO:teuthology.orchestra.run.smithi066.stdout:{"epoch":1,"fsid":"ed6470e2-8508-11ee-8dbe-212e2dc638e7","modified":"2023-11-17T05:20:13.934038Z","created":"2023-11-17T05:20:13.934038Z","min_mon_release":16,"min_mon_release_name":"pacific","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging"],"optional":[]},"mons":[{"rank":0,"name":"smithi040","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:3300","nonce":0},{"type":"v1","addr":"172.21.15.40:6789","nonce":0}]},"addr":"172.21.15.40:6789/0","public_addr":"172.21.15.40:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2023-11-17T05:31:34.281 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:34.280+0000 7fa750208700 1 -- 172.21.15.66:0/2487202475 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fa73402ffb0 msgr2=0x7fa734032470 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:31:34.281 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:34.280+0000 7fa750208700 1 --2- 172.21.15.66:0/2487202475 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fa73402ffb0 0x7fa734032470 secure :-1 s=READY pgs=134 cs=0 l=1 rev1=1 rx=0x7fa7380096a0 tx=0x7fa738006b40).stop 2023-11-17T05:31:34.282 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:34.280+0000 7fa750208700 1 -- 172.21.15.66:0/2487202475 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fa7480f5410 msgr2=0x7fa748109450 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:31:34.282 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:34.280+0000 7fa750208700 1 --2- 172.21.15.66:0/2487202475 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fa7480f5410 0x7fa748109450 secure :-1 s=READY pgs=338 cs=0 l=1 rev1=1 rx=0x7fa744013510 tx=0x7fa74400b040).stop 2023-11-17T05:31:34.282 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:34.280+0000 7fa750208700 1 -- 172.21.15.66:0/2487202475 shutdown_connections 2023-11-17T05:31:34.282 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:34.280+0000 7fa750208700 1 --2- 172.21.15.66:0/2487202475 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fa73402ffb0 0x7fa734032470 unknown :-1 s=CLOSED pgs=134 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:31:34.282 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:34.280+0000 7fa750208700 1 --2- 172.21.15.66:0/2487202475 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fa7480f5410 0x7fa748109450 unknown :-1 s=CLOSED pgs=338 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:31:34.282 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:34.280+0000 7fa750208700 1 -- 172.21.15.66:0/2487202475 >> 172.21.15.66:0/2487202475 conn(0x7fa7480f0b70 msgr2=0x7fa7480f1660 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:31:34.282 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:34.280+0000 7fa750208700 1 -- 172.21.15.66:0/2487202475 shutdown_connections 2023-11-17T05:31:34.282 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:34.280+0000 7fa750208700 1 -- 172.21.15.66:0/2487202475 wait complete. 2023-11-17T05:31:34.282 INFO:teuthology.orchestra.run.smithi066.stderr:dumped monmap epoch 1 2023-11-17T05:31:34.989 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:31:34 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:31:34.279790+0000 mon.smithi040 (mon.0) 279 : audit [DBG] from='client.? 172.21.15.66:0/2487202475' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-11-17T05:31:35.673 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2023-11-17T05:31:35.673 DEBUG:teuthology.orchestra.run.smithi066:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.4 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ed6470e2-8508-11ee-8dbe-212e2dc638e7 -- ceph mon dump -f json 2023-11-17T05:31:35.988 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:31:35 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:31:35.516532+0000 mgr.smithi040.ogcprl (mgr.14164) 323 : cluster [DBG] pgmap v307: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:31:36.885 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:36.883+0000 7fa9ad082700 1 -- 172.21.15.66:0/930443556 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fa9a80f6510 msgr2=0x7fa9a80f6930 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:31:36.885 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:36.883+0000 7fa9ad082700 1 --2- 172.21.15.66:0/930443556 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fa9a80f6510 0x7fa9a80f6930 secure :-1 s=READY pgs=339 cs=0 l=1 rev1=1 rx=0x7fa998009fb0 tx=0x7fa998009b30).stop 2023-11-17T05:31:36.885 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:36.883+0000 7fa9ad082700 1 -- 172.21.15.66:0/930443556 shutdown_connections 2023-11-17T05:31:36.886 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:36.883+0000 7fa9ad082700 1 --2- 172.21.15.66:0/930443556 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fa9a80f6510 0x7fa9a80f6930 unknown :-1 s=CLOSED pgs=339 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:31:36.886 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:36.883+0000 7fa9ad082700 1 -- 172.21.15.66:0/930443556 >> 172.21.15.66:0/930443556 conn(0x7fa9a80f1c70 msgr2=0x7fa9a80f4090 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:31:36.886 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:36.883+0000 7fa9ad082700 1 -- 172.21.15.66:0/930443556 shutdown_connections 2023-11-17T05:31:36.886 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:36.883+0000 7fa9ad082700 1 -- 172.21.15.66:0/930443556 wait complete. 2023-11-17T05:31:36.886 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:36.884+0000 7fa9ad082700 1 Processor -- start 2023-11-17T05:31:36.886 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:36.884+0000 7fa9ad082700 1 -- start start 2023-11-17T05:31:36.886 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:36.884+0000 7fa9ad082700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fa9a80f6510 0x7fa9a8106730 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:31:36.886 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:36.884+0000 7fa9ad082700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fa9a8106c20 con 0x7fa9a80f6510 2023-11-17T05:31:36.887 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:36.885+0000 7fa9a6d9d700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fa9a80f6510 0x7fa9a8106730 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:31:36.887 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:36.885+0000 7fa9a6d9d700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fa9a80f6510 0x7fa9a8106730 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.66:49900/0 (socket says 172.21.15.66:49900) 2023-11-17T05:31:36.887 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:36.885+0000 7fa9a6d9d700 1 -- 172.21.15.66:0/2443075820 learned_addr learned my addr 172.21.15.66:0/2443075820 (peer_addr_for_me v2:172.21.15.66:0/0) 2023-11-17T05:31:36.887 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:36.885+0000 7fa9a6d9d700 1 -- 172.21.15.66:0/2443075820 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fa998005040 con 0x7fa9a80f6510 2023-11-17T05:31:36.887 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:36.885+0000 7fa9a6d9d700 1 --2- 172.21.15.66:0/2443075820 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fa9a80f6510 0x7fa9a8106730 secure :-1 s=READY pgs=340 cs=0 l=1 rev1=1 rx=0x7fa998013510 tx=0x7fa99800b040).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:31:36.887 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:36.885+0000 7fa99f7fe700 1 -- 172.21.15.66:0/2443075820 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7fa998009320 con 0x7fa9a80f6510 2023-11-17T05:31:36.888 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:36.885+0000 7fa99f7fe700 1 -- 172.21.15.66:0/2443075820 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(25 keys) v1 ==== 1074+0+0 (secure 0 0 0) 0x7fa998013f00 con 0x7fa9a80f6510 2023-11-17T05:31:36.888 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:36.885+0000 7fa9ad082700 1 -- 172.21.15.66:0/2443075820 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fa9a8106e20 con 0x7fa9a80f6510 2023-11-17T05:31:36.888 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:36.885+0000 7fa99f7fe700 1 -- 172.21.15.66:0/2443075820 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7fa998014d90 con 0x7fa9a80f6510 2023-11-17T05:31:36.888 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:36.885+0000 7fa9ad082700 1 -- 172.21.15.66:0/2443075820 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fa9a81072b0 con 0x7fa9a80f6510 2023-11-17T05:31:36.889 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:36.886+0000 7fa99f7fe700 1 -- 172.21.15.66:0/2443075820 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 17) v1 ==== 36269+0+0 (secure 0 0 0) 0x7fa998014420 con 0x7fa9a80f6510 2023-11-17T05:31:36.889 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:36.887+0000 7fa99f7fe700 1 --2- 172.21.15.66:0/2443075820 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fa99402ff60 0x7fa994032420 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:31:36.889 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:36.887+0000 7fa99f7fe700 1 -- 172.21.15.66:0/2443075820 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(5..5 src has 1..5) v4 ==== 1270+0+0 (secure 0 0 0) 0x7fa998041110 con 0x7fa9a80f6510 2023-11-17T05:31:36.889 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:36.887+0000 7fa9ad082700 1 -- 172.21.15.66:0/2443075820 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fa9a8059100 con 0x7fa9a80f6510 2023-11-17T05:31:36.889 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:36.887+0000 7fa9a659c700 1 --2- 172.21.15.66:0/2443075820 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fa99402ff60 0x7fa994032420 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:31:36.889 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:36.887+0000 7fa9a659c700 1 --2- 172.21.15.66:0/2443075820 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fa99402ff60 0x7fa994032420 secure :-1 s=READY pgs=135 cs=0 l=1 rev1=1 rx=0x7fa9900096a0 tx=0x7fa990006b40).ready entity=mgr.14164 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:31:36.893 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:36.892+0000 7fa99f7fe700 1 -- 172.21.15.66:0/2443075820 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+152212 (secure 0 0 0) 0x7fa9980095f0 con 0x7fa9a80f6510 2023-11-17T05:31:37.112 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:37.111+0000 7fa9ad082700 1 -- 172.21.15.66:0/2443075820 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "mon dump", "format": "json"} v 0) v1 -- 0x7fa9a8045ad0 con 0x7fa9a80f6510 2023-11-17T05:31:37.112 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:37.111+0000 7fa99f7fe700 1 -- 172.21.15.66:0/2443075820 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mon_command_ack([{"prefix": "mon dump", "format": "json"}]=0 dumped monmap epoch 1 v1) v1 ==== 95+0+690 (secure 0 0 0) 0x7fa99801db70 con 0x7fa9a80f6510 2023-11-17T05:31:37.113 INFO:teuthology.orchestra.run.smithi066.stdout: 2023-11-17T05:31:37.113 INFO:teuthology.orchestra.run.smithi066.stdout:{"epoch":1,"fsid":"ed6470e2-8508-11ee-8dbe-212e2dc638e7","modified":"2023-11-17T05:20:13.934038Z","created":"2023-11-17T05:20:13.934038Z","min_mon_release":16,"min_mon_release_name":"pacific","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging"],"optional":[]},"mons":[{"rank":0,"name":"smithi040","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:3300","nonce":0},{"type":"v1","addr":"172.21.15.40:6789","nonce":0}]},"addr":"172.21.15.40:6789/0","public_addr":"172.21.15.40:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2023-11-17T05:31:37.114 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:37.113+0000 7fa9ad082700 1 -- 172.21.15.66:0/2443075820 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fa99402ff60 msgr2=0x7fa994032420 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:31:37.114 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:37.113+0000 7fa9ad082700 1 --2- 172.21.15.66:0/2443075820 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fa99402ff60 0x7fa994032420 secure :-1 s=READY pgs=135 cs=0 l=1 rev1=1 rx=0x7fa9900096a0 tx=0x7fa990006b40).stop 2023-11-17T05:31:37.114 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:37.113+0000 7fa9ad082700 1 -- 172.21.15.66:0/2443075820 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fa9a80f6510 msgr2=0x7fa9a8106730 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:31:37.115 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:37.113+0000 7fa9ad082700 1 --2- 172.21.15.66:0/2443075820 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fa9a80f6510 0x7fa9a8106730 secure :-1 s=READY pgs=340 cs=0 l=1 rev1=1 rx=0x7fa998013510 tx=0x7fa99800b040).stop 2023-11-17T05:31:37.115 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:37.113+0000 7fa9ad082700 1 -- 172.21.15.66:0/2443075820 shutdown_connections 2023-11-17T05:31:37.115 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:37.113+0000 7fa9ad082700 1 --2- 172.21.15.66:0/2443075820 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fa99402ff60 0x7fa994032420 unknown :-1 s=CLOSED pgs=135 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:31:37.115 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:37.113+0000 7fa9ad082700 1 --2- 172.21.15.66:0/2443075820 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fa9a80f6510 0x7fa9a8106730 unknown :-1 s=CLOSED pgs=340 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:31:37.115 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:37.113+0000 7fa9ad082700 1 -- 172.21.15.66:0/2443075820 >> 172.21.15.66:0/2443075820 conn(0x7fa9a80f1c70 msgr2=0x7fa9a80f2760 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:31:37.115 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:37.113+0000 7fa9ad082700 1 -- 172.21.15.66:0/2443075820 shutdown_connections 2023-11-17T05:31:37.115 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:37.113+0000 7fa9ad082700 1 -- 172.21.15.66:0/2443075820 wait complete. 2023-11-17T05:31:37.115 INFO:teuthology.orchestra.run.smithi066.stderr:dumped monmap epoch 1 2023-11-17T05:31:37.488 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:31:37 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:31:37.112933+0000 mon.smithi040 (mon.0) 280 : audit [DBG] from='client.? 172.21.15.66:0/2443075820' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-11-17T05:31:38.488 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:31:38 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:31:37.516885+0000 mgr.smithi040.ogcprl (mgr.14164) 324 : cluster [DBG] pgmap v308: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:31:38.505 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2023-11-17T05:31:38.505 DEBUG:teuthology.orchestra.run.smithi066:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.4 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ed6470e2-8508-11ee-8dbe-212e2dc638e7 -- ceph mon dump -f json 2023-11-17T05:31:39.611 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:39.609+0000 7f9a22ed0700 1 -- 172.21.15.66:0/925970756 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f9a1c060280 msgr2=0x7f9a1c0f90f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:31:39.611 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:39.609+0000 7f9a22ed0700 1 --2- 172.21.15.66:0/925970756 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f9a1c060280 0x7f9a1c0f90f0 secure :-1 s=READY pgs=341 cs=0 l=1 rev1=1 rx=0x7f9a10009fb0 tx=0x7f9a10009b30).stop 2023-11-17T05:31:39.611 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:39.609+0000 7f9a22ed0700 1 -- 172.21.15.66:0/925970756 shutdown_connections 2023-11-17T05:31:39.611 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:39.609+0000 7f9a22ed0700 1 --2- 172.21.15.66:0/925970756 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f9a1c060280 0x7f9a1c0f90f0 unknown :-1 s=CLOSED pgs=341 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:31:39.611 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:39.609+0000 7f9a22ed0700 1 -- 172.21.15.66:0/925970756 >> 172.21.15.66:0/925970756 conn(0x7f9a1c0f0b70 msgr2=0x7f9a1c0f2f90 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:31:39.611 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:39.609+0000 7f9a22ed0700 1 -- 172.21.15.66:0/925970756 shutdown_connections 2023-11-17T05:31:39.611 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:39.609+0000 7f9a22ed0700 1 -- 172.21.15.66:0/925970756 wait complete. 2023-11-17T05:31:39.611 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:39.610+0000 7f9a22ed0700 1 Processor -- start 2023-11-17T05:31:39.612 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:39.610+0000 7f9a22ed0700 1 -- start start 2023-11-17T05:31:39.613 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:39.610+0000 7f9a22ed0700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f9a1c060280 0x7f9a1c1066e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:31:39.613 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:39.610+0000 7f9a22ed0700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f9a1c106bd0 con 0x7f9a1c060280 2023-11-17T05:31:39.613 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:39.610+0000 7f9a20c6c700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f9a1c060280 0x7f9a1c1066e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:31:39.613 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:39.610+0000 7f9a20c6c700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f9a1c060280 0x7f9a1c1066e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.66:49924/0 (socket says 172.21.15.66:49924) 2023-11-17T05:31:39.613 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:39.610+0000 7f9a20c6c700 1 -- 172.21.15.66:0/1926135143 learned_addr learned my addr 172.21.15.66:0/1926135143 (peer_addr_for_me v2:172.21.15.66:0/0) 2023-11-17T05:31:39.613 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:39.611+0000 7f9a20c6c700 1 -- 172.21.15.66:0/1926135143 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f9a10005040 con 0x7f9a1c060280 2023-11-17T05:31:39.613 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:39.611+0000 7f9a20c6c700 1 --2- 172.21.15.66:0/1926135143 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f9a1c060280 0x7f9a1c1066e0 secure :-1 s=READY pgs=342 cs=0 l=1 rev1=1 rx=0x7f9a10013510 tx=0x7f9a1000b040).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:31:39.613 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:39.611+0000 7f9a197fa700 1 -- 172.21.15.66:0/1926135143 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f9a10009320 con 0x7f9a1c060280 2023-11-17T05:31:39.614 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:39.611+0000 7f9a22ed0700 1 -- 172.21.15.66:0/1926135143 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f9a1c106dd0 con 0x7f9a1c060280 2023-11-17T05:31:39.614 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:39.611+0000 7f9a197fa700 1 -- 172.21.15.66:0/1926135143 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(25 keys) v1 ==== 1074+0+0 (secure 0 0 0) 0x7f9a10013f00 con 0x7f9a1c060280 2023-11-17T05:31:39.614 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:39.611+0000 7f9a197fa700 1 -- 172.21.15.66:0/1926135143 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f9a10014dc0 con 0x7f9a1c060280 2023-11-17T05:31:39.614 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:39.611+0000 7f9a22ed0700 1 -- 172.21.15.66:0/1926135143 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f9a1c107260 con 0x7f9a1c060280 2023-11-17T05:31:39.614 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:39.612+0000 7f9a197fa700 1 -- 172.21.15.66:0/1926135143 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 17) v1 ==== 36269+0+0 (secure 0 0 0) 0x7f9a100095f0 con 0x7f9a1c060280 2023-11-17T05:31:39.615 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:39.612+0000 7f9a22ed0700 1 -- 172.21.15.66:0/1926135143 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f9a1c045ad0 con 0x7f9a1c060280 2023-11-17T05:31:39.615 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:39.612+0000 7f9a197fa700 1 --2- 172.21.15.66:0/1926135143 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f9a0c02ff60 0x7f9a0c032420 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:31:39.615 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:39.612+0000 7f9a197fa700 1 -- 172.21.15.66:0/1926135143 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(5..5 src has 1..5) v4 ==== 1270+0+0 (secure 0 0 0) 0x7f9a10041000 con 0x7f9a1c060280 2023-11-17T05:31:39.615 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:39.612+0000 7f9a1bfff700 1 --2- 172.21.15.66:0/1926135143 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f9a0c02ff60 0x7f9a0c032420 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:31:39.615 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:39.613+0000 7f9a1bfff700 1 --2- 172.21.15.66:0/1926135143 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f9a0c02ff60 0x7f9a0c032420 secure :-1 s=READY pgs=136 cs=0 l=1 rev1=1 rx=0x7f9a080096a0 tx=0x7f9a08006b40).ready entity=mgr.14164 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:31:39.622 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:39.617+0000 7f9a197fa700 1 -- 172.21.15.66:0/1926135143 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+152212 (secure 0 0 0) 0x7f9a10014420 con 0x7f9a1c060280 2023-11-17T05:31:39.837 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:39.835+0000 7f9a22ed0700 1 -- 172.21.15.66:0/1926135143 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "mon dump", "format": "json"} v 0) v1 -- 0x7f9a1c108110 con 0x7f9a1c060280 2023-11-17T05:31:39.837 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:39.836+0000 7f9a197fa700 1 -- 172.21.15.66:0/1926135143 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mon_command_ack([{"prefix": "mon dump", "format": "json"}]=0 dumped monmap epoch 1 v1) v1 ==== 95+0+690 (secure 0 0 0) 0x7f9a10010230 con 0x7f9a1c060280 2023-11-17T05:31:39.837 INFO:teuthology.orchestra.run.smithi066.stdout: 2023-11-17T05:31:39.838 INFO:teuthology.orchestra.run.smithi066.stdout:{"epoch":1,"fsid":"ed6470e2-8508-11ee-8dbe-212e2dc638e7","modified":"2023-11-17T05:20:13.934038Z","created":"2023-11-17T05:20:13.934038Z","min_mon_release":16,"min_mon_release_name":"pacific","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging"],"optional":[]},"mons":[{"rank":0,"name":"smithi040","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:3300","nonce":0},{"type":"v1","addr":"172.21.15.40:6789","nonce":0}]},"addr":"172.21.15.40:6789/0","public_addr":"172.21.15.40:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2023-11-17T05:31:39.839 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:39.838+0000 7f9a22ed0700 1 -- 172.21.15.66:0/1926135143 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f9a0c02ff60 msgr2=0x7f9a0c032420 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:31:39.839 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:39.838+0000 7f9a22ed0700 1 --2- 172.21.15.66:0/1926135143 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f9a0c02ff60 0x7f9a0c032420 secure :-1 s=READY pgs=136 cs=0 l=1 rev1=1 rx=0x7f9a080096a0 tx=0x7f9a08006b40).stop 2023-11-17T05:31:39.839 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:39.838+0000 7f9a22ed0700 1 -- 172.21.15.66:0/1926135143 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f9a1c060280 msgr2=0x7f9a1c1066e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:31:39.840 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:39.838+0000 7f9a22ed0700 1 --2- 172.21.15.66:0/1926135143 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f9a1c060280 0x7f9a1c1066e0 secure :-1 s=READY pgs=342 cs=0 l=1 rev1=1 rx=0x7f9a10013510 tx=0x7f9a1000b040).stop 2023-11-17T05:31:39.840 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:39.838+0000 7f9a22ed0700 1 -- 172.21.15.66:0/1926135143 shutdown_connections 2023-11-17T05:31:39.840 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:39.838+0000 7f9a22ed0700 1 --2- 172.21.15.66:0/1926135143 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f9a0c02ff60 0x7f9a0c032420 unknown :-1 s=CLOSED pgs=136 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:31:39.840 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:39.838+0000 7f9a22ed0700 1 --2- 172.21.15.66:0/1926135143 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f9a1c060280 0x7f9a1c1066e0 unknown :-1 s=CLOSED pgs=342 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:31:39.840 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:39.838+0000 7f9a22ed0700 1 -- 172.21.15.66:0/1926135143 >> 172.21.15.66:0/1926135143 conn(0x7f9a1c0f0b70 msgr2=0x7f9a1c0f1660 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:31:39.840 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:39.838+0000 7f9a22ed0700 1 -- 172.21.15.66:0/1926135143 shutdown_connections 2023-11-17T05:31:39.840 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:39.838+0000 7f9a22ed0700 1 -- 172.21.15.66:0/1926135143 wait complete. 2023-11-17T05:31:39.840 INFO:teuthology.orchestra.run.smithi066.stderr:dumped monmap epoch 1 2023-11-17T05:31:40.988 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:31:40 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:31:39.517235+0000 mgr.smithi040.ogcprl (mgr.14164) 325 : cluster [DBG] pgmap v309: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:31:40.989 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:31:40 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:31:39.837780+0000 mon.smithi040 (mon.0) 281 : audit [DBG] from='client.? 172.21.15.66:0/1926135143' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-11-17T05:31:41.215 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2023-11-17T05:31:41.215 DEBUG:teuthology.orchestra.run.smithi066:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.4 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ed6470e2-8508-11ee-8dbe-212e2dc638e7 -- ceph mon dump -f json 2023-11-17T05:31:41.990 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:31:41 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:31:41.517586+0000 mgr.smithi040.ogcprl (mgr.14164) 326 : cluster [DBG] pgmap v310: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:31:42.430 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:42.428+0000 7f912b59e700 1 -- 172.21.15.66:0/1902849516 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f912c0f78e0 msgr2=0x7f912c0f7d00 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:31:42.430 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:42.428+0000 7f912b59e700 1 --2- 172.21.15.66:0/1902849516 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f912c0f78e0 0x7f912c0f7d00 secure :-1 s=READY pgs=343 cs=0 l=1 rev1=1 rx=0x7f911c009fb0 tx=0x7f911c009b30).stop 2023-11-17T05:31:42.430 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:42.428+0000 7f912b59e700 1 -- 172.21.15.66:0/1902849516 shutdown_connections 2023-11-17T05:31:42.430 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:42.428+0000 7f912b59e700 1 --2- 172.21.15.66:0/1902849516 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f912c0f78e0 0x7f912c0f7d00 unknown :-1 s=CLOSED pgs=343 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:31:42.430 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:42.428+0000 7f912b59e700 1 -- 172.21.15.66:0/1902849516 >> 172.21.15.66:0/1902849516 conn(0x7f912c0f0120 msgr2=0x7f912c0f2540 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:31:42.430 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:42.428+0000 7f912b59e700 1 -- 172.21.15.66:0/1902849516 shutdown_connections 2023-11-17T05:31:42.430 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:42.428+0000 7f912b59e700 1 -- 172.21.15.66:0/1902849516 wait complete. 2023-11-17T05:31:42.431 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:42.428+0000 7f912b59e700 1 Processor -- start 2023-11-17T05:31:42.431 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:42.428+0000 7f912b59e700 1 -- start start 2023-11-17T05:31:42.431 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:42.429+0000 7f912b59e700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f912c0f78e0 0x7f912c1064e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:31:42.431 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:42.429+0000 7f912b59e700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f912c1069d0 con 0x7f912c0f78e0 2023-11-17T05:31:42.431 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:42.429+0000 7f912a59c700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f912c0f78e0 0x7f912c1064e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:31:42.431 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:42.429+0000 7f912a59c700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f912c0f78e0 0x7f912c1064e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.66:53504/0 (socket says 172.21.15.66:53504) 2023-11-17T05:31:42.431 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:42.429+0000 7f912a59c700 1 -- 172.21.15.66:0/2688867683 learned_addr learned my addr 172.21.15.66:0/2688867683 (peer_addr_for_me v2:172.21.15.66:0/0) 2023-11-17T05:31:42.432 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:42.429+0000 7f912a59c700 1 -- 172.21.15.66:0/2688867683 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f911c005040 con 0x7f912c0f78e0 2023-11-17T05:31:42.432 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:42.429+0000 7f912a59c700 1 --2- 172.21.15.66:0/2688867683 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f912c0f78e0 0x7f912c1064e0 secure :-1 s=READY pgs=344 cs=0 l=1 rev1=1 rx=0x7f911c013510 tx=0x7f911c00b040).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:31:42.433 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:42.429+0000 7f911affd700 1 -- 172.21.15.66:0/2688867683 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f911c009320 con 0x7f912c0f78e0 2023-11-17T05:31:42.433 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:42.429+0000 7f912b59e700 1 -- 172.21.15.66:0/2688867683 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f912c106bd0 con 0x7f912c0f78e0 2023-11-17T05:31:42.433 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:42.430+0000 7f911affd700 1 -- 172.21.15.66:0/2688867683 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(25 keys) v1 ==== 1074+0+0 (secure 0 0 0) 0x7f911c013f00 con 0x7f912c0f78e0 2023-11-17T05:31:42.433 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:42.430+0000 7f911affd700 1 -- 172.21.15.66:0/2688867683 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f911c014dc0 con 0x7f912c0f78e0 2023-11-17T05:31:42.433 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:42.430+0000 7f912b59e700 1 -- 172.21.15.66:0/2688867683 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f912c107060 con 0x7f912c0f78e0 2023-11-17T05:31:42.433 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:42.431+0000 7f911affd700 1 -- 172.21.15.66:0/2688867683 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 17) v1 ==== 36269+0+0 (secure 0 0 0) 0x7f911c0095f0 con 0x7f912c0f78e0 2023-11-17T05:31:42.433 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:42.431+0000 7f912b59e700 1 -- 172.21.15.66:0/2688867683 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f912c045ad0 con 0x7f912c0f78e0 2023-11-17T05:31:42.433 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:42.431+0000 7f911affd700 1 --2- 172.21.15.66:0/2688867683 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f911402ff60 0x7f9114032420 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:31:42.433 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:42.431+0000 7f911affd700 1 -- 172.21.15.66:0/2688867683 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(5..5 src has 1..5) v4 ==== 1270+0+0 (secure 0 0 0) 0x7f911c041000 con 0x7f912c0f78e0 2023-11-17T05:31:42.433 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:42.431+0000 7f9129d9b700 1 --2- 172.21.15.66:0/2688867683 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f911402ff60 0x7f9114032420 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:31:42.433 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:42.431+0000 7f9129d9b700 1 --2- 172.21.15.66:0/2688867683 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f911402ff60 0x7f9114032420 secure :-1 s=READY pgs=137 cs=0 l=1 rev1=1 rx=0x7f91200096a0 tx=0x7f9120006b40).ready entity=mgr.14164 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:31:42.439 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:42.435+0000 7f911affd700 1 -- 172.21.15.66:0/2688867683 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+152212 (secure 0 0 0) 0x7f911c014420 con 0x7f912c0f78e0 2023-11-17T05:31:42.656 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:42.654+0000 7f912b59e700 1 -- 172.21.15.66:0/2688867683 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "mon dump", "format": "json"} v 0) v1 -- 0x7f912c107f10 con 0x7f912c0f78e0 2023-11-17T05:31:42.656 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:42.655+0000 7f911affd700 1 -- 172.21.15.66:0/2688867683 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mon_command_ack([{"prefix": "mon dump", "format": "json"}]=0 dumped monmap epoch 1 v1) v1 ==== 95+0+690 (secure 0 0 0) 0x7f911c010230 con 0x7f912c0f78e0 2023-11-17T05:31:42.657 INFO:teuthology.orchestra.run.smithi066.stdout: 2023-11-17T05:31:42.657 INFO:teuthology.orchestra.run.smithi066.stdout:{"epoch":1,"fsid":"ed6470e2-8508-11ee-8dbe-212e2dc638e7","modified":"2023-11-17T05:20:13.934038Z","created":"2023-11-17T05:20:13.934038Z","min_mon_release":16,"min_mon_release_name":"pacific","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging"],"optional":[]},"mons":[{"rank":0,"name":"smithi040","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:3300","nonce":0},{"type":"v1","addr":"172.21.15.40:6789","nonce":0}]},"addr":"172.21.15.40:6789/0","public_addr":"172.21.15.40:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2023-11-17T05:31:42.658 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:42.657+0000 7f912b59e700 1 -- 172.21.15.66:0/2688867683 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f911402ff60 msgr2=0x7f9114032420 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:31:42.658 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:42.657+0000 7f912b59e700 1 --2- 172.21.15.66:0/2688867683 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f911402ff60 0x7f9114032420 secure :-1 s=READY pgs=137 cs=0 l=1 rev1=1 rx=0x7f91200096a0 tx=0x7f9120006b40).stop 2023-11-17T05:31:42.658 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:42.657+0000 7f912b59e700 1 -- 172.21.15.66:0/2688867683 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f912c0f78e0 msgr2=0x7f912c1064e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:31:42.658 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:42.657+0000 7f912b59e700 1 --2- 172.21.15.66:0/2688867683 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f912c0f78e0 0x7f912c1064e0 secure :-1 s=READY pgs=344 cs=0 l=1 rev1=1 rx=0x7f911c013510 tx=0x7f911c00b040).stop 2023-11-17T05:31:42.659 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:42.657+0000 7f912b59e700 1 -- 172.21.15.66:0/2688867683 shutdown_connections 2023-11-17T05:31:42.659 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:42.657+0000 7f912b59e700 1 --2- 172.21.15.66:0/2688867683 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f911402ff60 0x7f9114032420 unknown :-1 s=CLOSED pgs=137 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:31:42.659 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:42.657+0000 7f912b59e700 1 --2- 172.21.15.66:0/2688867683 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f912c0f78e0 0x7f912c1064e0 unknown :-1 s=CLOSED pgs=344 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:31:42.659 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:42.657+0000 7f912b59e700 1 -- 172.21.15.66:0/2688867683 >> 172.21.15.66:0/2688867683 conn(0x7f912c0f0120 msgr2=0x7f912c0f0ba0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:31:42.659 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:42.657+0000 7f912b59e700 1 -- 172.21.15.66:0/2688867683 shutdown_connections 2023-11-17T05:31:42.659 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:42.657+0000 7f912b59e700 1 -- 172.21.15.66:0/2688867683 wait complete. 2023-11-17T05:31:42.659 INFO:teuthology.orchestra.run.smithi066.stderr:dumped monmap epoch 1 2023-11-17T05:31:42.989 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:31:42 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:31:42.656889+0000 mon.smithi040 (mon.0) 282 : audit [DBG] from='client.? 172.21.15.66:0/2688867683' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-11-17T05:31:43.989 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:31:43 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:31:43.517935+0000 mgr.smithi040.ogcprl (mgr.14164) 327 : cluster [DBG] pgmap v311: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:31:44.049 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2023-11-17T05:31:44.050 DEBUG:teuthology.orchestra.run.smithi066:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.4 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ed6470e2-8508-11ee-8dbe-212e2dc638e7 -- ceph mon dump -f json 2023-11-17T05:31:45.250 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:45.248+0000 7fe405aa9700 1 -- 172.21.15.66:0/1400469092 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fe400060280 msgr2=0x7fe4000f90f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:31:45.250 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:45.248+0000 7fe405aa9700 1 --2- 172.21.15.66:0/1400469092 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fe400060280 0x7fe4000f90f0 secure :-1 s=READY pgs=345 cs=0 l=1 rev1=1 rx=0x7fe3f0009fb0 tx=0x7fe3f0009b30).stop 2023-11-17T05:31:45.250 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:45.248+0000 7fe405aa9700 1 -- 172.21.15.66:0/1400469092 shutdown_connections 2023-11-17T05:31:45.250 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:45.248+0000 7fe405aa9700 1 --2- 172.21.15.66:0/1400469092 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fe400060280 0x7fe4000f90f0 unknown :-1 s=CLOSED pgs=345 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:31:45.250 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:45.248+0000 7fe405aa9700 1 -- 172.21.15.66:0/1400469092 >> 172.21.15.66:0/1400469092 conn(0x7fe4000f0b70 msgr2=0x7fe4000f2f90 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:31:45.250 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:45.248+0000 7fe405aa9700 1 -- 172.21.15.66:0/1400469092 shutdown_connections 2023-11-17T05:31:45.250 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:45.248+0000 7fe405aa9700 1 -- 172.21.15.66:0/1400469092 wait complete. 2023-11-17T05:31:45.251 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:45.249+0000 7fe405aa9700 1 Processor -- start 2023-11-17T05:31:45.251 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:45.249+0000 7fe405aa9700 1 -- start start 2023-11-17T05:31:45.251 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:45.249+0000 7fe405aa9700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fe400060280 0x7fe4001066e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:31:45.251 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:45.249+0000 7fe405aa9700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fe400106bd0 con 0x7fe400060280 2023-11-17T05:31:45.252 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:45.250+0000 7fe3ff7fe700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fe400060280 0x7fe4001066e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:31:45.252 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:45.250+0000 7fe3ff7fe700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fe400060280 0x7fe4001066e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.66:53520/0 (socket says 172.21.15.66:53520) 2023-11-17T05:31:45.252 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:45.250+0000 7fe3ff7fe700 1 -- 172.21.15.66:0/4142050172 learned_addr learned my addr 172.21.15.66:0/4142050172 (peer_addr_for_me v2:172.21.15.66:0/0) 2023-11-17T05:31:45.252 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:45.250+0000 7fe3ff7fe700 1 -- 172.21.15.66:0/4142050172 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fe3f0005040 con 0x7fe400060280 2023-11-17T05:31:45.252 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:45.250+0000 7fe3ff7fe700 1 --2- 172.21.15.66:0/4142050172 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fe400060280 0x7fe4001066e0 secure :-1 s=READY pgs=346 cs=0 l=1 rev1=1 rx=0x7fe3f0013510 tx=0x7fe3f000b040).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:31:45.253 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:45.250+0000 7fe3e7fff700 1 -- 172.21.15.66:0/4142050172 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7fe3f0009320 con 0x7fe400060280 2023-11-17T05:31:45.253 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:45.251+0000 7fe405aa9700 1 -- 172.21.15.66:0/4142050172 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fe400106dd0 con 0x7fe400060280 2023-11-17T05:31:45.253 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:45.251+0000 7fe3e7fff700 1 -- 172.21.15.66:0/4142050172 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(25 keys) v1 ==== 1074+0+0 (secure 0 0 0) 0x7fe3f0013f00 con 0x7fe400060280 2023-11-17T05:31:45.253 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:45.251+0000 7fe3e7fff700 1 -- 172.21.15.66:0/4142050172 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7fe3f0014dc0 con 0x7fe400060280 2023-11-17T05:31:45.253 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:45.251+0000 7fe405aa9700 1 -- 172.21.15.66:0/4142050172 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fe400107260 con 0x7fe400060280 2023-11-17T05:31:45.253 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:45.251+0000 7fe3e7fff700 1 -- 172.21.15.66:0/4142050172 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 17) v1 ==== 36269+0+0 (secure 0 0 0) 0x7fe3f0014420 con 0x7fe400060280 2023-11-17T05:31:45.253 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:45.251+0000 7fe405aa9700 1 -- 172.21.15.66:0/4142050172 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fe400045ad0 con 0x7fe400060280 2023-11-17T05:31:45.254 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:45.251+0000 7fe3e7fff700 1 --2- 172.21.15.66:0/4142050172 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fe3e802ff60 0x7fe3e8032420 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:31:45.254 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:45.252+0000 7fe3e7fff700 1 -- 172.21.15.66:0/4142050172 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(5..5 src has 1..5) v4 ==== 1270+0+0 (secure 0 0 0) 0x7fe3f0041030 con 0x7fe400060280 2023-11-17T05:31:45.254 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:45.252+0000 7fe3feffd700 1 --2- 172.21.15.66:0/4142050172 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fe3e802ff60 0x7fe3e8032420 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:31:45.254 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:45.252+0000 7fe3feffd700 1 --2- 172.21.15.66:0/4142050172 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fe3e802ff60 0x7fe3e8032420 secure :-1 s=READY pgs=138 cs=0 l=1 rev1=1 rx=0x7fe3f40096a0 tx=0x7fe3f4006b40).ready entity=mgr.14164 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:31:45.257 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:45.256+0000 7fe3e7fff700 1 -- 172.21.15.66:0/4142050172 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+152212 (secure 0 0 0) 0x7fe3f00095f0 con 0x7fe400060280 2023-11-17T05:31:45.475 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:45.473+0000 7fe405aa9700 1 -- 172.21.15.66:0/4142050172 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "mon dump", "format": "json"} v 0) v1 -- 0x7fe400108110 con 0x7fe400060280 2023-11-17T05:31:45.475 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:45.474+0000 7fe3e7fff700 1 -- 172.21.15.66:0/4142050172 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mon_command_ack([{"prefix": "mon dump", "format": "json"}]=0 dumped monmap epoch 1 v1) v1 ==== 95+0+690 (secure 0 0 0) 0x7fe3f001cb10 con 0x7fe400060280 2023-11-17T05:31:45.476 INFO:teuthology.orchestra.run.smithi066.stdout: 2023-11-17T05:31:45.476 INFO:teuthology.orchestra.run.smithi066.stdout:{"epoch":1,"fsid":"ed6470e2-8508-11ee-8dbe-212e2dc638e7","modified":"2023-11-17T05:20:13.934038Z","created":"2023-11-17T05:20:13.934038Z","min_mon_release":16,"min_mon_release_name":"pacific","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging"],"optional":[]},"mons":[{"rank":0,"name":"smithi040","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:3300","nonce":0},{"type":"v1","addr":"172.21.15.40:6789","nonce":0}]},"addr":"172.21.15.40:6789/0","public_addr":"172.21.15.40:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2023-11-17T05:31:45.477 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:45.475+0000 7fe405aa9700 1 -- 172.21.15.66:0/4142050172 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fe3e802ff60 msgr2=0x7fe3e8032420 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:31:45.477 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:45.475+0000 7fe405aa9700 1 --2- 172.21.15.66:0/4142050172 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fe3e802ff60 0x7fe3e8032420 secure :-1 s=READY pgs=138 cs=0 l=1 rev1=1 rx=0x7fe3f40096a0 tx=0x7fe3f4006b40).stop 2023-11-17T05:31:45.477 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:45.475+0000 7fe405aa9700 1 -- 172.21.15.66:0/4142050172 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fe400060280 msgr2=0x7fe4001066e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:31:45.477 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:45.475+0000 7fe405aa9700 1 --2- 172.21.15.66:0/4142050172 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fe400060280 0x7fe4001066e0 secure :-1 s=READY pgs=346 cs=0 l=1 rev1=1 rx=0x7fe3f0013510 tx=0x7fe3f000b040).stop 2023-11-17T05:31:45.477 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:45.475+0000 7fe405aa9700 1 -- 172.21.15.66:0/4142050172 shutdown_connections 2023-11-17T05:31:45.477 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:45.475+0000 7fe405aa9700 1 --2- 172.21.15.66:0/4142050172 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fe3e802ff60 0x7fe3e8032420 unknown :-1 s=CLOSED pgs=138 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:31:45.477 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:45.475+0000 7fe405aa9700 1 --2- 172.21.15.66:0/4142050172 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fe400060280 0x7fe4001066e0 unknown :-1 s=CLOSED pgs=346 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:31:45.477 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:45.475+0000 7fe405aa9700 1 -- 172.21.15.66:0/4142050172 >> 172.21.15.66:0/4142050172 conn(0x7fe4000f0b70 msgr2=0x7fe4000f1660 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:31:45.477 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:45.476+0000 7fe405aa9700 1 -- 172.21.15.66:0/4142050172 shutdown_connections 2023-11-17T05:31:45.477 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:45.476+0000 7fe405aa9700 1 -- 172.21.15.66:0/4142050172 wait complete. 2023-11-17T05:31:45.478 INFO:teuthology.orchestra.run.smithi066.stderr:dumped monmap epoch 1 2023-11-17T05:31:45.990 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:31:45 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:31:45.475751+0000 mon.smithi040 (mon.0) 283 : audit [DBG] from='client.? 172.21.15.66:0/4142050172' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-11-17T05:31:46.895 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2023-11-17T05:31:46.957 DEBUG:teuthology.orchestra.run.smithi066:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.4 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ed6470e2-8508-11ee-8dbe-212e2dc638e7 -- ceph mon dump -f json 2023-11-17T05:31:47.034 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:31:46 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:31:45.518318+0000 mgr.smithi040.ogcprl (mgr.14164) 328 : cluster [DBG] pgmap v312: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:31:47.989 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:31:47 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:31:47.518631+0000 mgr.smithi040.ogcprl (mgr.14164) 329 : cluster [DBG] pgmap v313: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:31:48.251 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:48.249+0000 7f1a82c09700 1 -- 172.21.15.66:0/872911437 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f1a7c0f62a0 msgr2=0x7f1a7c0f66c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:31:48.252 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:48.249+0000 7f1a82c09700 1 --2- 172.21.15.66:0/872911437 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f1a7c0f62a0 0x7f1a7c0f66c0 secure :-1 s=READY pgs=347 cs=0 l=1 rev1=1 rx=0x7f1a78009fb0 tx=0x7f1a78009b30).stop 2023-11-17T05:31:48.252 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:48.249+0000 7f1a82c09700 1 -- 172.21.15.66:0/872911437 shutdown_connections 2023-11-17T05:31:48.252 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:48.249+0000 7f1a82c09700 1 --2- 172.21.15.66:0/872911437 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f1a7c0f62a0 0x7f1a7c0f66c0 unknown :-1 s=CLOSED pgs=347 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:31:48.252 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:48.249+0000 7f1a82c09700 1 -- 172.21.15.66:0/872911437 >> 172.21.15.66:0/872911437 conn(0x7f1a7c0f1aa0 msgr2=0x7f1a7c0f3ec0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:31:48.252 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:48.249+0000 7f1a82c09700 1 -- 172.21.15.66:0/872911437 shutdown_connections 2023-11-17T05:31:48.252 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:48.249+0000 7f1a82c09700 1 -- 172.21.15.66:0/872911437 wait complete. 2023-11-17T05:31:48.252 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:48.249+0000 7f1a82c09700 1 Processor -- start 2023-11-17T05:31:48.252 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:48.249+0000 7f1a82c09700 1 -- start start 2023-11-17T05:31:48.252 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:48.249+0000 7f1a82c09700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f1a7c0f62a0 0x7f1a7c1064b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:31:48.252 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:48.249+0000 7f1a82c09700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f1a7c1069a0 con 0x7f1a7c0f62a0 2023-11-17T05:31:48.252 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:48.250+0000 7f1a81c07700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f1a7c0f62a0 0x7f1a7c1064b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:31:48.252 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:48.250+0000 7f1a81c07700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f1a7c0f62a0 0x7f1a7c1064b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.66:53542/0 (socket says 172.21.15.66:53542) 2023-11-17T05:31:48.253 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:48.250+0000 7f1a81c07700 1 -- 172.21.15.66:0/575599957 learned_addr learned my addr 172.21.15.66:0/575599957 (peer_addr_for_me v2:172.21.15.66:0/0) 2023-11-17T05:31:48.253 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:48.250+0000 7f1a81c07700 1 -- 172.21.15.66:0/575599957 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f1a78005040 con 0x7f1a7c0f62a0 2023-11-17T05:31:48.253 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:48.250+0000 7f1a81c07700 1 --2- 172.21.15.66:0/575599957 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f1a7c0f62a0 0x7f1a7c1064b0 secure :-1 s=READY pgs=348 cs=0 l=1 rev1=1 rx=0x7f1a78013510 tx=0x7f1a7800b040).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:31:48.253 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:48.250+0000 7f1a727fc700 1 -- 172.21.15.66:0/575599957 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f1a78009320 con 0x7f1a7c0f62a0 2023-11-17T05:31:48.253 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:48.250+0000 7f1a82c09700 1 -- 172.21.15.66:0/575599957 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f1a7c106ba0 con 0x7f1a7c0f62a0 2023-11-17T05:31:48.253 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:48.250+0000 7f1a727fc700 1 -- 172.21.15.66:0/575599957 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(25 keys) v1 ==== 1074+0+0 (secure 0 0 0) 0x7f1a78013f00 con 0x7f1a7c0f62a0 2023-11-17T05:31:48.253 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:48.250+0000 7f1a727fc700 1 -- 172.21.15.66:0/575599957 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f1a78014dc0 con 0x7f1a7c0f62a0 2023-11-17T05:31:48.253 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:48.250+0000 7f1a82c09700 1 -- 172.21.15.66:0/575599957 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f1a7c107030 con 0x7f1a7c0f62a0 2023-11-17T05:31:48.254 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:48.251+0000 7f1a727fc700 1 -- 172.21.15.66:0/575599957 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 17) v1 ==== 36269+0+0 (secure 0 0 0) 0x7f1a780095f0 con 0x7f1a7c0f62a0 2023-11-17T05:31:48.254 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:48.251+0000 7f1a82c09700 1 -- 172.21.15.66:0/575599957 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f1a7c045ad0 con 0x7f1a7c0f62a0 2023-11-17T05:31:48.254 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:48.251+0000 7f1a727fc700 1 --2- 172.21.15.66:0/575599957 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f1a6802ff60 0x7f1a68032420 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:31:48.254 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:48.251+0000 7f1a727fc700 1 -- 172.21.15.66:0/575599957 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(5..5 src has 1..5) v4 ==== 1270+0+0 (secure 0 0 0) 0x7f1a780410c0 con 0x7f1a7c0f62a0 2023-11-17T05:31:48.254 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:48.251+0000 7f1a81406700 1 --2- 172.21.15.66:0/575599957 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f1a6802ff60 0x7f1a68032420 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:31:48.254 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:48.252+0000 7f1a81406700 1 --2- 172.21.15.66:0/575599957 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f1a6802ff60 0x7f1a68032420 secure :-1 s=READY pgs=139 cs=0 l=1 rev1=1 rx=0x7f1a6c0096a0 tx=0x7f1a6c006b40).ready entity=mgr.14164 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:31:48.257 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:48.256+0000 7f1a727fc700 1 -- 172.21.15.66:0/575599957 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+152212 (secure 0 0 0) 0x7f1a78014420 con 0x7f1a7c0f62a0 2023-11-17T05:31:48.476 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:48.475+0000 7f1a82c09700 1 -- 172.21.15.66:0/575599957 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "mon dump", "format": "json"} v 0) v1 -- 0x7f1a7c107ee0 con 0x7f1a7c0f62a0 2023-11-17T05:31:48.477 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:48.475+0000 7f1a727fc700 1 -- 172.21.15.66:0/575599957 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mon_command_ack([{"prefix": "mon dump", "format": "json"}]=0 dumped monmap epoch 1 v1) v1 ==== 95+0+690 (secure 0 0 0) 0x7f1a78010230 con 0x7f1a7c0f62a0 2023-11-17T05:31:48.477 INFO:teuthology.orchestra.run.smithi066.stdout: 2023-11-17T05:31:48.477 INFO:teuthology.orchestra.run.smithi066.stdout:{"epoch":1,"fsid":"ed6470e2-8508-11ee-8dbe-212e2dc638e7","modified":"2023-11-17T05:20:13.934038Z","created":"2023-11-17T05:20:13.934038Z","min_mon_release":16,"min_mon_release_name":"pacific","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging"],"optional":[]},"mons":[{"rank":0,"name":"smithi040","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:3300","nonce":0},{"type":"v1","addr":"172.21.15.40:6789","nonce":0}]},"addr":"172.21.15.40:6789/0","public_addr":"172.21.15.40:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2023-11-17T05:31:48.478 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:48.477+0000 7f1a82c09700 1 -- 172.21.15.66:0/575599957 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f1a6802ff60 msgr2=0x7f1a68032420 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:31:48.478 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:48.477+0000 7f1a82c09700 1 --2- 172.21.15.66:0/575599957 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f1a6802ff60 0x7f1a68032420 secure :-1 s=READY pgs=139 cs=0 l=1 rev1=1 rx=0x7f1a6c0096a0 tx=0x7f1a6c006b40).stop 2023-11-17T05:31:48.479 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:48.477+0000 7f1a82c09700 1 -- 172.21.15.66:0/575599957 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f1a7c0f62a0 msgr2=0x7f1a7c1064b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:31:48.479 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:48.477+0000 7f1a82c09700 1 --2- 172.21.15.66:0/575599957 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f1a7c0f62a0 0x7f1a7c1064b0 secure :-1 s=READY pgs=348 cs=0 l=1 rev1=1 rx=0x7f1a78013510 tx=0x7f1a7800b040).stop 2023-11-17T05:31:48.479 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:48.477+0000 7f1a82c09700 1 -- 172.21.15.66:0/575599957 shutdown_connections 2023-11-17T05:31:48.479 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:48.477+0000 7f1a82c09700 1 --2- 172.21.15.66:0/575599957 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f1a6802ff60 0x7f1a68032420 unknown :-1 s=CLOSED pgs=139 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:31:48.479 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:48.477+0000 7f1a82c09700 1 --2- 172.21.15.66:0/575599957 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f1a7c0f62a0 0x7f1a7c1064b0 unknown :-1 s=CLOSED pgs=348 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:31:48.479 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:48.477+0000 7f1a82c09700 1 -- 172.21.15.66:0/575599957 >> 172.21.15.66:0/575599957 conn(0x7f1a7c0f1aa0 msgr2=0x7f1a7c0f2510 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:31:48.479 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:48.477+0000 7f1a82c09700 1 -- 172.21.15.66:0/575599957 shutdown_connections 2023-11-17T05:31:48.479 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:48.477+0000 7f1a82c09700 1 -- 172.21.15.66:0/575599957 wait complete. 2023-11-17T05:31:48.480 INFO:teuthology.orchestra.run.smithi066.stderr:dumped monmap epoch 1 2023-11-17T05:31:48.988 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:31:48 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:31:48.476942+0000 mon.smithi040 (mon.0) 284 : audit [DBG] from='client.? 172.21.15.66:0/575599957' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-11-17T05:31:49.887 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2023-11-17T05:31:49.887 DEBUG:teuthology.orchestra.run.smithi066:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.4 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ed6470e2-8508-11ee-8dbe-212e2dc638e7 -- ceph mon dump -f json 2023-11-17T05:31:50.989 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:31:50 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:31:49.518937+0000 mgr.smithi040.ogcprl (mgr.14164) 330 : cluster [DBG] pgmap v314: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:31:59.579 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:31:51 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:31:51.308864+0000 mon.smithi040 (mon.0) 285 : audit [DBG] from='client.? 172.21.15.66:0/2900243462' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-11-17T05:31:59.580 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:31:52 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:31:51.519310+0000 mgr.smithi040.ogcprl (mgr.14164) 331 : cluster [DBG] pgmap v315: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:31:59.580 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:31:53 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:31:53.519755+0000 mgr.smithi040.ogcprl (mgr.14164) 332 : cluster [DBG] pgmap v316: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:31:59.580 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:31:55 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:31:55.520127+0000 mgr.smithi040.ogcprl (mgr.14164) 333 : cluster [DBG] pgmap v317: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:31:59.580 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:31:57 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:31:57.520467+0000 mgr.smithi040.ogcprl (mgr.14164) 334 : cluster [DBG] pgmap v318: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:31:59.582 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:51.081+0000 7fbf00a54700 1 -- 172.21.15.66:0/2226821398 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fbefc0f5410 msgr2=0x7fbefc0f5830 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:31:59.582 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:51.081+0000 7fbf00a54700 1 --2- 172.21.15.66:0/2226821398 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fbefc0f5410 0x7fbefc0f5830 secure :-1 s=READY pgs=349 cs=0 l=1 rev1=1 rx=0x7fbef0009fb0 tx=0x7fbef0009b30).stop 2023-11-17T05:31:59.662 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:51.081+0000 7fbf00a54700 1 -- 172.21.15.66:0/2226821398 shutdown_connections 2023-11-17T05:31:59.662 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:51.081+0000 7fbf00a54700 1 --2- 172.21.15.66:0/2226821398 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fbefc0f5410 0x7fbefc0f5830 unknown :-1 s=CLOSED pgs=349 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:31:59.662 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:51.081+0000 7fbf00a54700 1 -- 172.21.15.66:0/2226821398 >> 172.21.15.66:0/2226821398 conn(0x7fbefc0f0b70 msgr2=0x7fbefc0f2f90 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:31:59.662 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:51.081+0000 7fbf00a54700 1 -- 172.21.15.66:0/2226821398 shutdown_connections 2023-11-17T05:31:59.662 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:51.081+0000 7fbf00a54700 1 -- 172.21.15.66:0/2226821398 wait complete. 2023-11-17T05:31:59.662 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:51.082+0000 7fbf00a54700 1 Processor -- start 2023-11-17T05:31:59.662 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:51.082+0000 7fbf00a54700 1 -- start start 2023-11-17T05:31:59.662 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:51.082+0000 7fbf00a54700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fbefc0f5410 0x7fbefc10b830 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:31:59.662 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:51.082+0000 7fbf00a54700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fbefc107220 con 0x7fbefc0f5410 2023-11-17T05:31:59.662 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:51.083+0000 7fbefa59c700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fbefc0f5410 0x7fbefc10b830 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:31:59.662 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:51.083+0000 7fbefa59c700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fbefc0f5410 0x7fbefc10b830 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.66:53564/0 (socket says 172.21.15.66:53564) 2023-11-17T05:31:59.662 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:51.083+0000 7fbefa59c700 1 -- 172.21.15.66:0/2900243462 learned_addr learned my addr 172.21.15.66:0/2900243462 (peer_addr_for_me v2:172.21.15.66:0/0) 2023-11-17T05:31:59.663 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:51.083+0000 7fbefa59c700 1 -- 172.21.15.66:0/2900243462 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fbef0005040 con 0x7fbefc0f5410 2023-11-17T05:31:59.663 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:51.083+0000 7fbefa59c700 1 --2- 172.21.15.66:0/2900243462 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fbefc0f5410 0x7fbefc10b830 secure :-1 s=READY pgs=350 cs=0 l=1 rev1=1 rx=0x7fbef0013510 tx=0x7fbef000b040).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:31:59.663 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:51.083+0000 7fbeeaffd700 1 -- 172.21.15.66:0/2900243462 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7fbef0009320 con 0x7fbefc0f5410 2023-11-17T05:31:59.663 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:51.083+0000 7fbeeaffd700 1 -- 172.21.15.66:0/2900243462 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(25 keys) v1 ==== 1074+0+0 (secure 0 0 0) 0x7fbef0013f00 con 0x7fbefc0f5410 2023-11-17T05:31:59.663 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:51.083+0000 7fbf00a54700 1 -- 172.21.15.66:0/2900243462 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fbefc060240 con 0x7fbefc0f5410 2023-11-17T05:31:59.663 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:51.083+0000 7fbeeaffd700 1 -- 172.21.15.66:0/2900243462 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7fbef0014d90 con 0x7fbefc0f5410 2023-11-17T05:31:59.663 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:51.083+0000 7fbf00a54700 1 -- 172.21.15.66:0/2900243462 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fbefc10bd20 con 0x7fbefc0f5410 2023-11-17T05:31:59.663 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:51.084+0000 7fbeeaffd700 1 -- 172.21.15.66:0/2900243462 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 17) v1 ==== 36269+0+0 (secure 0 0 0) 0x7fbef0014420 con 0x7fbefc0f5410 2023-11-17T05:31:59.663 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:51.084+0000 7fbf00a54700 1 -- 172.21.15.66:0/2900243462 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fbefc045ad0 con 0x7fbefc0f5410 2023-11-17T05:31:59.663 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:51.084+0000 7fbeeaffd700 1 --2- 172.21.15.66:0/2900243462 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fbee402ffb0 0x7fbee4032470 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:31:59.663 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:51.084+0000 7fbeeaffd700 1 -- 172.21.15.66:0/2900243462 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(5..5 src has 1..5) v4 ==== 1270+0+0 (secure 0 0 0) 0x7fbef0041090 con 0x7fbefc0f5410 2023-11-17T05:31:59.663 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:51.085+0000 7fbef9d9b700 1 --2- 172.21.15.66:0/2900243462 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fbee402ffb0 0x7fbee4032470 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:31:59.663 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:51.085+0000 7fbef9d9b700 1 --2- 172.21.15.66:0/2900243462 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fbee402ffb0 0x7fbee4032470 secure :-1 s=READY pgs=140 cs=0 l=1 rev1=1 rx=0x7fbeec0096a0 tx=0x7fbeec006b40).ready entity=mgr.14164 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:31:59.664 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:51.089+0000 7fbeeaffd700 1 -- 172.21.15.66:0/2900243462 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+152212 (secure 0 0 0) 0x7fbef006d2b0 con 0x7fbefc0f5410 2023-11-17T05:31:59.664 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:51.307+0000 7fbf00a54700 1 -- 172.21.15.66:0/2900243462 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "mon dump", "format": "json"} v 0) v1 -- 0x7fbefc0fae60 con 0x7fbefc0f5410 2023-11-17T05:31:59.664 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:51.307+0000 7fbeeaffd700 1 -- 172.21.15.66:0/2900243462 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mon_command_ack([{"prefix": "mon dump", "format": "json"}]=0 dumped monmap epoch 1 v1) v1 ==== 95+0+690 (secure 0 0 0) 0x7fbef00095f0 con 0x7fbefc0f5410 2023-11-17T05:31:59.664 INFO:teuthology.orchestra.run.smithi066.stdout: 2023-11-17T05:31:59.664 INFO:teuthology.orchestra.run.smithi066.stdout:{"epoch":1,"fsid":"ed6470e2-8508-11ee-8dbe-212e2dc638e7","modified":"2023-11-17T05:20:13.934038Z","created":"2023-11-17T05:20:13.934038Z","min_mon_release":16,"min_mon_release_name":"pacific","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging"],"optional":[]},"mons":[{"rank":0,"name":"smithi040","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:3300","nonce":0},{"type":"v1","addr":"172.21.15.40:6789","nonce":0}]},"addr":"172.21.15.40:6789/0","public_addr":"172.21.15.40:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2023-11-17T05:31:59.665 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:51.309+0000 7fbf00a54700 1 -- 172.21.15.66:0/2900243462 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fbee402ffb0 msgr2=0x7fbee4032470 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:31:59.665 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:51.309+0000 7fbf00a54700 1 --2- 172.21.15.66:0/2900243462 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fbee402ffb0 0x7fbee4032470 secure :-1 s=READY pgs=140 cs=0 l=1 rev1=1 rx=0x7fbeec0096a0 tx=0x7fbeec006b40).stop 2023-11-17T05:31:59.665 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:51.309+0000 7fbf00a54700 1 -- 172.21.15.66:0/2900243462 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fbefc0f5410 msgr2=0x7fbefc10b830 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:31:59.666 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:51.309+0000 7fbf00a54700 1 --2- 172.21.15.66:0/2900243462 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fbefc0f5410 0x7fbefc10b830 secure :-1 s=READY pgs=350 cs=0 l=1 rev1=1 rx=0x7fbef0013510 tx=0x7fbef000b040).stop 2023-11-17T05:31:59.666 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:51.309+0000 7fbf00a54700 1 -- 172.21.15.66:0/2900243462 shutdown_connections 2023-11-17T05:31:59.666 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:51.309+0000 7fbf00a54700 1 --2- 172.21.15.66:0/2900243462 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fbee402ffb0 0x7fbee4032470 unknown :-1 s=CLOSED pgs=140 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:31:59.666 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:51.309+0000 7fbf00a54700 1 --2- 172.21.15.66:0/2900243462 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fbefc0f5410 0x7fbefc10b830 unknown :-1 s=CLOSED pgs=350 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:31:59.666 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:51.309+0000 7fbf00a54700 1 -- 172.21.15.66:0/2900243462 >> 172.21.15.66:0/2900243462 conn(0x7fbefc0f0b70 msgr2=0x7fbefc0f1660 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:31:59.666 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:51.309+0000 7fbf00a54700 1 -- 172.21.15.66:0/2900243462 shutdown_connections 2023-11-17T05:31:59.666 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:31:51.309+0000 7fbf00a54700 1 -- 172.21.15.66:0/2900243462 wait complete. 2023-11-17T05:31:59.666 INFO:teuthology.orchestra.run.smithi066.stderr:dumped monmap epoch 1 2023-11-17T05:32:00.667 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2023-11-17T05:32:00.667 DEBUG:teuthology.orchestra.run.smithi066:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.4 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ed6470e2-8508-11ee-8dbe-212e2dc638e7 -- ceph mon dump -f json 2023-11-17T05:32:00.988 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:32:00 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:31:59.520811+0000 mgr.smithi040.ogcprl (mgr.14164) 335 : cluster [DBG] pgmap v319: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:32:01.872 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:01.870+0000 7f67a4489700 1 -- 172.21.15.66:0/1530466415 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f679c0f5410 msgr2=0x7f679c0f5830 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:32:01.873 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:01.870+0000 7f67a4489700 1 --2- 172.21.15.66:0/1530466415 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f679c0f5410 0x7f679c0f5830 secure :-1 s=READY pgs=351 cs=0 l=1 rev1=1 rx=0x7f6798009fb0 tx=0x7f6798009b30).stop 2023-11-17T05:32:01.873 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:01.870+0000 7f67a4489700 1 -- 172.21.15.66:0/1530466415 shutdown_connections 2023-11-17T05:32:01.873 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:01.870+0000 7f67a4489700 1 --2- 172.21.15.66:0/1530466415 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f679c0f5410 0x7f679c0f5830 unknown :-1 s=CLOSED pgs=351 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:32:01.873 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:01.870+0000 7f67a4489700 1 -- 172.21.15.66:0/1530466415 >> 172.21.15.66:0/1530466415 conn(0x7f679c0f0b70 msgr2=0x7f679c0f2f90 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:32:01.873 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:01.870+0000 7f67a4489700 1 -- 172.21.15.66:0/1530466415 shutdown_connections 2023-11-17T05:32:01.873 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:01.870+0000 7f67a4489700 1 -- 172.21.15.66:0/1530466415 wait complete. 2023-11-17T05:32:01.873 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:01.871+0000 7f67a4489700 1 Processor -- start 2023-11-17T05:32:01.873 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:01.871+0000 7f67a4489700 1 -- start start 2023-11-17T05:32:01.873 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:01.871+0000 7f67a4489700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f679c0f5410 0x7f679c0fd3a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:32:01.874 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:01.871+0000 7f67a4489700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f679c0fb820 con 0x7f679c0f5410 2023-11-17T05:32:01.874 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:01.871+0000 7f67a2225700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f679c0f5410 0x7f679c0fd3a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:32:01.874 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:01.871+0000 7f67a2225700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f679c0f5410 0x7f679c0fd3a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.66:50036/0 (socket says 172.21.15.66:50036) 2023-11-17T05:32:01.874 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:01.871+0000 7f67a2225700 1 -- 172.21.15.66:0/2750223478 learned_addr learned my addr 172.21.15.66:0/2750223478 (peer_addr_for_me v2:172.21.15.66:0/0) 2023-11-17T05:32:01.874 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:01.872+0000 7f67a2225700 1 -- 172.21.15.66:0/2750223478 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f6798005040 con 0x7f679c0f5410 2023-11-17T05:32:01.874 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:01.872+0000 7f67a2225700 1 --2- 172.21.15.66:0/2750223478 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f679c0f5410 0x7f679c0fd3a0 secure :-1 s=READY pgs=352 cs=0 l=1 rev1=1 rx=0x7f6798013510 tx=0x7f679800b040).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:32:01.874 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:01.872+0000 7f678effd700 1 -- 172.21.15.66:0/2750223478 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f6798009320 con 0x7f679c0f5410 2023-11-17T05:32:01.874 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:01.872+0000 7f67a4489700 1 -- 172.21.15.66:0/2750223478 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f679c0fb9c0 con 0x7f679c0f5410 2023-11-17T05:32:01.874 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:01.872+0000 7f678effd700 1 -- 172.21.15.66:0/2750223478 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(25 keys) v1 ==== 1074+0+0 (secure 0 0 0) 0x7f6798013f00 con 0x7f679c0f5410 2023-11-17T05:32:01.874 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:01.872+0000 7f678effd700 1 -- 172.21.15.66:0/2750223478 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f6798014d90 con 0x7f679c0f5410 2023-11-17T05:32:01.874 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:01.872+0000 7f67a4489700 1 -- 172.21.15.66:0/2750223478 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f679c0fbe50 con 0x7f679c0f5410 2023-11-17T05:32:01.875 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:01.873+0000 7f67a4489700 1 -- 172.21.15.66:0/2750223478 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f679c059100 con 0x7f679c0f5410 2023-11-17T05:32:01.876 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:01.873+0000 7f678effd700 1 -- 172.21.15.66:0/2750223478 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 17) v1 ==== 36269+0+0 (secure 0 0 0) 0x7f67980095f0 con 0x7f679c0f5410 2023-11-17T05:32:01.876 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:01.873+0000 7f678effd700 1 --2- 172.21.15.66:0/2750223478 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f678802ff60 0x7f6788032420 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:32:01.876 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:01.873+0000 7f678effd700 1 -- 172.21.15.66:0/2750223478 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(5..5 src has 1..5) v4 ==== 1270+0+0 (secure 0 0 0) 0x7f6798040fc0 con 0x7f679c0f5410 2023-11-17T05:32:01.876 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:01.873+0000 7f67a1a24700 1 --2- 172.21.15.66:0/2750223478 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f678802ff60 0x7f6788032420 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:32:01.876 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:01.873+0000 7f67a1a24700 1 --2- 172.21.15.66:0/2750223478 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f678802ff60 0x7f6788032420 secure :-1 s=READY pgs=141 cs=0 l=1 rev1=1 rx=0x7f67900096a0 tx=0x7f6790006b40).ready entity=mgr.14164 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:32:01.878 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:01.878+0000 7f678effd700 1 -- 172.21.15.66:0/2750223478 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+152212 (secure 0 0 0) 0x7f6798014420 con 0x7f679c0f5410 2023-11-17T05:32:01.988 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:32:01 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:32:01.521169+0000 mgr.smithi040.ogcprl (mgr.14164) 336 : cluster [DBG] pgmap v320: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:32:02.099 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:02.098+0000 7f67a4489700 1 -- 172.21.15.66:0/2750223478 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "mon dump", "format": "json"} v 0) v1 -- 0x7f679c045ad0 con 0x7f679c0f5410 2023-11-17T05:32:02.099 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:02.099+0000 7f678effd700 1 -- 172.21.15.66:0/2750223478 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mon_command_ack([{"prefix": "mon dump", "format": "json"}]=0 dumped monmap epoch 1 v1) v1 ==== 95+0+690 (secure 0 0 0) 0x7f6798010230 con 0x7f679c0f5410 2023-11-17T05:32:02.100 INFO:teuthology.orchestra.run.smithi066.stdout: 2023-11-17T05:32:02.100 INFO:teuthology.orchestra.run.smithi066.stdout:{"epoch":1,"fsid":"ed6470e2-8508-11ee-8dbe-212e2dc638e7","modified":"2023-11-17T05:20:13.934038Z","created":"2023-11-17T05:20:13.934038Z","min_mon_release":16,"min_mon_release_name":"pacific","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging"],"optional":[]},"mons":[{"rank":0,"name":"smithi040","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:3300","nonce":0},{"type":"v1","addr":"172.21.15.40:6789","nonce":0}]},"addr":"172.21.15.40:6789/0","public_addr":"172.21.15.40:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2023-11-17T05:32:02.101 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:02.100+0000 7f67a4489700 1 -- 172.21.15.66:0/2750223478 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f678802ff60 msgr2=0x7f6788032420 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:32:02.101 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:02.100+0000 7f67a4489700 1 --2- 172.21.15.66:0/2750223478 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f678802ff60 0x7f6788032420 secure :-1 s=READY pgs=141 cs=0 l=1 rev1=1 rx=0x7f67900096a0 tx=0x7f6790006b40).stop 2023-11-17T05:32:02.101 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:02.100+0000 7f67a4489700 1 -- 172.21.15.66:0/2750223478 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f679c0f5410 msgr2=0x7f679c0fd3a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:32:02.101 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:02.100+0000 7f67a4489700 1 --2- 172.21.15.66:0/2750223478 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f679c0f5410 0x7f679c0fd3a0 secure :-1 s=READY pgs=352 cs=0 l=1 rev1=1 rx=0x7f6798013510 tx=0x7f679800b040).stop 2023-11-17T05:32:02.101 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:02.100+0000 7f67a4489700 1 -- 172.21.15.66:0/2750223478 shutdown_connections 2023-11-17T05:32:02.101 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:02.100+0000 7f67a4489700 1 --2- 172.21.15.66:0/2750223478 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f678802ff60 0x7f6788032420 unknown :-1 s=CLOSED pgs=141 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:32:02.101 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:02.100+0000 7f67a4489700 1 --2- 172.21.15.66:0/2750223478 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f679c0f5410 0x7f679c0fd3a0 unknown :-1 s=CLOSED pgs=352 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:32:02.101 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:02.101+0000 7f67a4489700 1 -- 172.21.15.66:0/2750223478 >> 172.21.15.66:0/2750223478 conn(0x7f679c0f0b70 msgr2=0x7f679c05fe20 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:32:02.101 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:02.101+0000 7f67a4489700 1 -- 172.21.15.66:0/2750223478 shutdown_connections 2023-11-17T05:32:02.102 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:02.101+0000 7f67a4489700 1 -- 172.21.15.66:0/2750223478 wait complete. 2023-11-17T05:32:02.102 INFO:teuthology.orchestra.run.smithi066.stderr:dumped monmap epoch 1 2023-11-17T05:32:02.989 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:32:02 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:32:02.099793+0000 mon.smithi040 (mon.0) 286 : audit [DBG] from='client.? 172.21.15.66:0/2750223478' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-11-17T05:32:03.480 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2023-11-17T05:32:03.481 DEBUG:teuthology.orchestra.run.smithi066:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.4 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ed6470e2-8508-11ee-8dbe-212e2dc638e7 -- ceph mon dump -f json 2023-11-17T05:32:03.989 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:32:03 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:32:03.521514+0000 mgr.smithi040.ogcprl (mgr.14164) 337 : cluster [DBG] pgmap v321: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:32:04.676 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:04.671+0000 7fa22c8b3700 1 -- 172.21.15.66:0/1020005806 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fa2280f5410 msgr2=0x7fa2280f5830 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:32:04.676 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:04.671+0000 7fa22c8b3700 1 --2- 172.21.15.66:0/1020005806 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fa2280f5410 0x7fa2280f5830 secure :-1 s=READY pgs=353 cs=0 l=1 rev1=1 rx=0x7fa218009fb0 tx=0x7fa218009b30).stop 2023-11-17T05:32:04.677 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:04.671+0000 7fa22c8b3700 1 -- 172.21.15.66:0/1020005806 shutdown_connections 2023-11-17T05:32:04.677 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:04.671+0000 7fa22c8b3700 1 --2- 172.21.15.66:0/1020005806 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fa2280f5410 0x7fa2280f5830 unknown :-1 s=CLOSED pgs=353 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:32:04.677 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:04.671+0000 7fa22c8b3700 1 -- 172.21.15.66:0/1020005806 >> 172.21.15.66:0/1020005806 conn(0x7fa2280f0b70 msgr2=0x7fa2280f2f90 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:32:04.677 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:04.671+0000 7fa22c8b3700 1 -- 172.21.15.66:0/1020005806 shutdown_connections 2023-11-17T05:32:04.677 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:04.671+0000 7fa22c8b3700 1 -- 172.21.15.66:0/1020005806 wait complete. 2023-11-17T05:32:04.678 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:04.672+0000 7fa22c8b3700 1 Processor -- start 2023-11-17T05:32:04.678 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:04.672+0000 7fa22c8b3700 1 -- start start 2023-11-17T05:32:04.678 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:04.672+0000 7fa22c8b3700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fa2280f5410 0x7fa2281093f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:32:04.678 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:04.672+0000 7fa22c8b3700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fa228060280 con 0x7fa2280f5410 2023-11-17T05:32:04.678 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:04.672+0000 7fa22659c700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fa2280f5410 0x7fa2281093f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:32:04.679 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:04.672+0000 7fa22659c700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fa2280f5410 0x7fa2281093f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.66:37752/0 (socket says 172.21.15.66:37752) 2023-11-17T05:32:04.679 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:04.672+0000 7fa22659c700 1 -- 172.21.15.66:0/2517331687 learned_addr learned my addr 172.21.15.66:0/2517331687 (peer_addr_for_me v2:172.21.15.66:0/0) 2023-11-17T05:32:04.679 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:04.673+0000 7fa22659c700 1 -- 172.21.15.66:0/2517331687 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fa218005040 con 0x7fa2280f5410 2023-11-17T05:32:04.679 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:04.673+0000 7fa22659c700 1 --2- 172.21.15.66:0/2517331687 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fa2280f5410 0x7fa2281093f0 secure :-1 s=READY pgs=354 cs=0 l=1 rev1=1 rx=0x7fa218013510 tx=0x7fa21800b040).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:32:04.679 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:04.673+0000 7fa216ffd700 1 -- 172.21.15.66:0/2517331687 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7fa218009320 con 0x7fa2280f5410 2023-11-17T05:32:04.680 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:04.673+0000 7fa22c8b3700 1 -- 172.21.15.66:0/2517331687 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fa228060480 con 0x7fa2280f5410 2023-11-17T05:32:04.680 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:04.673+0000 7fa216ffd700 1 -- 172.21.15.66:0/2517331687 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(25 keys) v1 ==== 1074+0+0 (secure 0 0 0) 0x7fa218013f00 con 0x7fa2280f5410 2023-11-17T05:32:04.680 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:04.673+0000 7fa216ffd700 1 -- 172.21.15.66:0/2517331687 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7fa218014d90 con 0x7fa2280f5410 2023-11-17T05:32:04.680 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:04.673+0000 7fa22c8b3700 1 -- 172.21.15.66:0/2517331687 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fa228109b20 con 0x7fa2280f5410 2023-11-17T05:32:04.680 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:04.674+0000 7fa216ffd700 1 -- 172.21.15.66:0/2517331687 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 17) v1 ==== 36269+0+0 (secure 0 0 0) 0x7fa2180095f0 con 0x7fa2280f5410 2023-11-17T05:32:04.680 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:04.674+0000 7fa22c8b3700 1 -- 172.21.15.66:0/2517331687 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fa228045ad0 con 0x7fa2280f5410 2023-11-17T05:32:04.681 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:04.674+0000 7fa216ffd700 1 --2- 172.21.15.66:0/2517331687 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fa21002ff60 0x7fa210032420 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:32:04.681 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:04.674+0000 7fa216ffd700 1 -- 172.21.15.66:0/2517331687 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(5..5 src has 1..5) v4 ==== 1270+0+0 (secure 0 0 0) 0x7fa218040fc0 con 0x7fa2280f5410 2023-11-17T05:32:04.681 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:04.674+0000 7fa225d9b700 1 --2- 172.21.15.66:0/2517331687 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fa21002ff60 0x7fa210032420 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:32:04.681 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:04.675+0000 7fa225d9b700 1 --2- 172.21.15.66:0/2517331687 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fa21002ff60 0x7fa210032420 secure :-1 s=READY pgs=142 cs=0 l=1 rev1=1 rx=0x7fa21c0096a0 tx=0x7fa21c006b40).ready entity=mgr.14164 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:32:04.682 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:04.679+0000 7fa216ffd700 1 -- 172.21.15.66:0/2517331687 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+152212 (secure 0 0 0) 0x7fa218014420 con 0x7fa2280f5410 2023-11-17T05:32:04.899 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:04.898+0000 7fa22c8b3700 1 -- 172.21.15.66:0/2517331687 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "mon dump", "format": "json"} v 0) v1 -- 0x7fa22810a370 con 0x7fa2280f5410 2023-11-17T05:32:04.900 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:04.898+0000 7fa216ffd700 1 -- 172.21.15.66:0/2517331687 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mon_command_ack([{"prefix": "mon dump", "format": "json"}]=0 dumped monmap epoch 1 v1) v1 ==== 95+0+690 (secure 0 0 0) 0x7fa218010230 con 0x7fa2280f5410 2023-11-17T05:32:04.900 INFO:teuthology.orchestra.run.smithi066.stdout: 2023-11-17T05:32:04.900 INFO:teuthology.orchestra.run.smithi066.stdout:{"epoch":1,"fsid":"ed6470e2-8508-11ee-8dbe-212e2dc638e7","modified":"2023-11-17T05:20:13.934038Z","created":"2023-11-17T05:20:13.934038Z","min_mon_release":16,"min_mon_release_name":"pacific","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging"],"optional":[]},"mons":[{"rank":0,"name":"smithi040","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:3300","nonce":0},{"type":"v1","addr":"172.21.15.40:6789","nonce":0}]},"addr":"172.21.15.40:6789/0","public_addr":"172.21.15.40:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2023-11-17T05:32:04.901 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:04.900+0000 7fa22c8b3700 1 -- 172.21.15.66:0/2517331687 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fa21002ff60 msgr2=0x7fa210032420 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:32:04.901 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:04.900+0000 7fa22c8b3700 1 --2- 172.21.15.66:0/2517331687 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fa21002ff60 0x7fa210032420 secure :-1 s=READY pgs=142 cs=0 l=1 rev1=1 rx=0x7fa21c0096a0 tx=0x7fa21c006b40).stop 2023-11-17T05:32:04.901 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:04.900+0000 7fa22c8b3700 1 -- 172.21.15.66:0/2517331687 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fa2280f5410 msgr2=0x7fa2281093f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:32:04.901 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:04.900+0000 7fa22c8b3700 1 --2- 172.21.15.66:0/2517331687 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fa2280f5410 0x7fa2281093f0 secure :-1 s=READY pgs=354 cs=0 l=1 rev1=1 rx=0x7fa218013510 tx=0x7fa21800b040).stop 2023-11-17T05:32:04.901 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:04.900+0000 7fa22c8b3700 1 -- 172.21.15.66:0/2517331687 shutdown_connections 2023-11-17T05:32:04.901 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:04.900+0000 7fa22c8b3700 1 --2- 172.21.15.66:0/2517331687 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fa21002ff60 0x7fa210032420 unknown :-1 s=CLOSED pgs=142 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:32:04.901 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:04.900+0000 7fa22c8b3700 1 --2- 172.21.15.66:0/2517331687 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fa2280f5410 0x7fa2281093f0 unknown :-1 s=CLOSED pgs=354 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:32:04.901 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:04.900+0000 7fa22c8b3700 1 -- 172.21.15.66:0/2517331687 >> 172.21.15.66:0/2517331687 conn(0x7fa2280f0b70 msgr2=0x7fa2280f1660 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:32:04.901 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:04.900+0000 7fa22c8b3700 1 -- 172.21.15.66:0/2517331687 shutdown_connections 2023-11-17T05:32:04.901 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:04.900+0000 7fa22c8b3700 1 -- 172.21.15.66:0/2517331687 wait complete. 2023-11-17T05:32:04.902 INFO:teuthology.orchestra.run.smithi066.stderr:dumped monmap epoch 1 2023-11-17T05:32:05.989 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:32:05 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:32:04.898801+0000 mon.smithi040 (mon.0) 287 : audit [DBG] from='client.? 172.21.15.66:0/2517331687' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-11-17T05:32:06.287 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2023-11-17T05:32:06.287 DEBUG:teuthology.orchestra.run.smithi066:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.4 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ed6470e2-8508-11ee-8dbe-212e2dc638e7 -- ceph mon dump -f json 2023-11-17T05:32:06.989 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:32:06 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:32:05.521906+0000 mgr.smithi040.ogcprl (mgr.14164) 338 : cluster [DBG] pgmap v322: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:32:07.467 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:07.463+0000 7f13d882b700 1 -- 172.21.15.66:0/3836433490 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f13d40f5410 msgr2=0x7f13d40f5830 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:32:07.468 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:07.463+0000 7f13d882b700 1 --2- 172.21.15.66:0/3836433490 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f13d40f5410 0x7f13d40f5830 secure :-1 s=READY pgs=355 cs=0 l=1 rev1=1 rx=0x7f13c8009fb0 tx=0x7f13c8009b30).stop 2023-11-17T05:32:07.468 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:07.463+0000 7f13d882b700 1 -- 172.21.15.66:0/3836433490 shutdown_connections 2023-11-17T05:32:07.468 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:07.463+0000 7f13d882b700 1 --2- 172.21.15.66:0/3836433490 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f13d40f5410 0x7f13d40f5830 unknown :-1 s=CLOSED pgs=355 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:32:07.468 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:07.463+0000 7f13d882b700 1 -- 172.21.15.66:0/3836433490 >> 172.21.15.66:0/3836433490 conn(0x7f13d40f0b70 msgr2=0x7f13d40f2f90 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:32:07.468 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:07.463+0000 7f13d882b700 1 -- 172.21.15.66:0/3836433490 shutdown_connections 2023-11-17T05:32:07.469 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:07.463+0000 7f13d882b700 1 -- 172.21.15.66:0/3836433490 wait complete. 2023-11-17T05:32:07.469 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:07.464+0000 7f13d882b700 1 Processor -- start 2023-11-17T05:32:07.469 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:07.464+0000 7f13d882b700 1 -- start start 2023-11-17T05:32:07.469 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:07.464+0000 7f13d882b700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f13d40f5410 0x7f13d41093f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:32:07.469 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:07.464+0000 7f13d882b700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f13d4060280 con 0x7f13d40f5410 2023-11-17T05:32:07.469 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:07.465+0000 7f13d259c700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f13d40f5410 0x7f13d41093f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:32:07.470 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:07.465+0000 7f13d259c700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f13d40f5410 0x7f13d41093f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.66:37784/0 (socket says 172.21.15.66:37784) 2023-11-17T05:32:07.470 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:07.465+0000 7f13d259c700 1 -- 172.21.15.66:0/894937489 learned_addr learned my addr 172.21.15.66:0/894937489 (peer_addr_for_me v2:172.21.15.66:0/0) 2023-11-17T05:32:07.470 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:07.465+0000 7f13d259c700 1 -- 172.21.15.66:0/894937489 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f13c8005040 con 0x7f13d40f5410 2023-11-17T05:32:07.470 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:07.465+0000 7f13d259c700 1 --2- 172.21.15.66:0/894937489 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f13d40f5410 0x7f13d41093f0 secure :-1 s=READY pgs=356 cs=0 l=1 rev1=1 rx=0x7f13c8013510 tx=0x7f13c800b040).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:32:07.470 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:07.465+0000 7f13c2ffd700 1 -- 172.21.15.66:0/894937489 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f13c8009320 con 0x7f13d40f5410 2023-11-17T05:32:07.470 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:07.465+0000 7f13d882b700 1 -- 172.21.15.66:0/894937489 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f13d4060480 con 0x7f13d40f5410 2023-11-17T05:32:07.471 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:07.465+0000 7f13c2ffd700 1 -- 172.21.15.66:0/894937489 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(25 keys) v1 ==== 1074+0+0 (secure 0 0 0) 0x7f13c8013f00 con 0x7f13d40f5410 2023-11-17T05:32:07.471 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:07.465+0000 7f13c2ffd700 1 -- 172.21.15.66:0/894937489 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f13c8014d90 con 0x7f13d40f5410 2023-11-17T05:32:07.471 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:07.465+0000 7f13d882b700 1 -- 172.21.15.66:0/894937489 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f13d4109b20 con 0x7f13d40f5410 2023-11-17T05:32:07.471 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:07.466+0000 7f13c2ffd700 1 -- 172.21.15.66:0/894937489 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 17) v1 ==== 36269+0+0 (secure 0 0 0) 0x7f13c80095f0 con 0x7f13d40f5410 2023-11-17T05:32:07.471 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:07.466+0000 7f13d882b700 1 -- 172.21.15.66:0/894937489 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f13d4045ad0 con 0x7f13d40f5410 2023-11-17T05:32:07.471 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:07.466+0000 7f13c2ffd700 1 --2- 172.21.15.66:0/894937489 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f13bc02ff60 0x7f13bc032420 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:32:07.472 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:07.466+0000 7f13c2ffd700 1 -- 172.21.15.66:0/894937489 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(5..5 src has 1..5) v4 ==== 1270+0+0 (secure 0 0 0) 0x7f13c8041000 con 0x7f13d40f5410 2023-11-17T05:32:07.472 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:07.467+0000 7f13d1d9b700 1 --2- 172.21.15.66:0/894937489 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f13bc02ff60 0x7f13bc032420 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:32:07.473 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:07.467+0000 7f13d1d9b700 1 --2- 172.21.15.66:0/894937489 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f13bc02ff60 0x7f13bc032420 secure :-1 s=READY pgs=143 cs=0 l=1 rev1=1 rx=0x7f13c40096a0 tx=0x7f13c4006b40).ready entity=mgr.14164 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:32:07.473 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:07.471+0000 7f13c2ffd700 1 -- 172.21.15.66:0/894937489 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+152212 (secure 0 0 0) 0x7f13c8014420 con 0x7f13d40f5410 2023-11-17T05:32:07.699 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:07.690+0000 7f13d882b700 1 -- 172.21.15.66:0/894937489 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "mon dump", "format": "json"} v 0) v1 -- 0x7f13d410a340 con 0x7f13d40f5410 2023-11-17T05:32:07.700 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:07.691+0000 7f13c2ffd700 1 -- 172.21.15.66:0/894937489 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mon_command_ack([{"prefix": "mon dump", "format": "json"}]=0 dumped monmap epoch 1 v1) v1 ==== 95+0+690 (secure 0 0 0) 0x7f13c8010230 con 0x7f13d40f5410 2023-11-17T05:32:07.702 INFO:teuthology.orchestra.run.smithi066.stdout: 2023-11-17T05:32:07.702 INFO:teuthology.orchestra.run.smithi066.stdout:{"epoch":1,"fsid":"ed6470e2-8508-11ee-8dbe-212e2dc638e7","modified":"2023-11-17T05:20:13.934038Z","created":"2023-11-17T05:20:13.934038Z","min_mon_release":16,"min_mon_release_name":"pacific","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging"],"optional":[]},"mons":[{"rank":0,"name":"smithi040","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:3300","nonce":0},{"type":"v1","addr":"172.21.15.40:6789","nonce":0}]},"addr":"172.21.15.40:6789/0","public_addr":"172.21.15.40:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2023-11-17T05:32:07.703 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:07.692+0000 7f13d882b700 1 -- 172.21.15.66:0/894937489 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f13bc02ff60 msgr2=0x7f13bc032420 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:32:07.703 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:07.692+0000 7f13d882b700 1 --2- 172.21.15.66:0/894937489 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f13bc02ff60 0x7f13bc032420 secure :-1 s=READY pgs=143 cs=0 l=1 rev1=1 rx=0x7f13c40096a0 tx=0x7f13c4006b40).stop 2023-11-17T05:32:07.703 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:07.692+0000 7f13d882b700 1 -- 172.21.15.66:0/894937489 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f13d40f5410 msgr2=0x7f13d41093f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:32:07.703 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:07.692+0000 7f13d882b700 1 --2- 172.21.15.66:0/894937489 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f13d40f5410 0x7f13d41093f0 secure :-1 s=READY pgs=356 cs=0 l=1 rev1=1 rx=0x7f13c8013510 tx=0x7f13c800b040).stop 2023-11-17T05:32:07.703 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:07.692+0000 7f13d882b700 1 -- 172.21.15.66:0/894937489 shutdown_connections 2023-11-17T05:32:07.703 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:07.692+0000 7f13d882b700 1 --2- 172.21.15.66:0/894937489 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f13bc02ff60 0x7f13bc032420 unknown :-1 s=CLOSED pgs=143 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:32:07.703 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:07.692+0000 7f13d882b700 1 --2- 172.21.15.66:0/894937489 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f13d40f5410 0x7f13d41093f0 unknown :-1 s=CLOSED pgs=356 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:32:07.703 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:07.692+0000 7f13d882b700 1 -- 172.21.15.66:0/894937489 >> 172.21.15.66:0/894937489 conn(0x7f13d40f0b70 msgr2=0x7f13d40f1660 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:32:07.704 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:07.692+0000 7f13d882b700 1 -- 172.21.15.66:0/894937489 shutdown_connections 2023-11-17T05:32:07.704 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:07.693+0000 7f13d882b700 1 -- 172.21.15.66:0/894937489 wait complete. 2023-11-17T05:32:07.704 INFO:teuthology.orchestra.run.smithi066.stderr:dumped monmap epoch 1 2023-11-17T05:32:07.989 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:32:07 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:32:07.522231+0000 mgr.smithi040.ogcprl (mgr.14164) 339 : cluster [DBG] pgmap v323: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:32:08.989 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:32:08 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:32:07.691397+0000 mon.smithi040 (mon.0) 288 : audit [DBG] from='client.? 172.21.15.66:0/894937489' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-11-17T05:32:09.123 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2023-11-17T05:32:09.124 DEBUG:teuthology.orchestra.run.smithi066:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.4 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ed6470e2-8508-11ee-8dbe-212e2dc638e7 -- ceph mon dump -f json 2023-11-17T05:32:10.239 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:10.238+0000 7fcec050c700 1 -- 172.21.15.66:0/1597146305 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fceb80f5410 msgr2=0x7fceb80f5830 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:32:10.239 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:10.238+0000 7fcec050c700 1 --2- 172.21.15.66:0/1597146305 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fceb80f5410 0x7fceb80f5830 secure :-1 s=READY pgs=357 cs=0 l=1 rev1=1 rx=0x7fceb4009fb0 tx=0x7fceb4009b30).stop 2023-11-17T05:32:10.239 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:10.238+0000 7fcec050c700 1 -- 172.21.15.66:0/1597146305 shutdown_connections 2023-11-17T05:32:10.239 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:10.238+0000 7fcec050c700 1 --2- 172.21.15.66:0/1597146305 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fceb80f5410 0x7fceb80f5830 unknown :-1 s=CLOSED pgs=357 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:32:10.239 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:10.238+0000 7fcec050c700 1 -- 172.21.15.66:0/1597146305 >> 172.21.15.66:0/1597146305 conn(0x7fceb80f0b70 msgr2=0x7fceb80f2f90 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:32:10.239 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:10.238+0000 7fcec050c700 1 -- 172.21.15.66:0/1597146305 shutdown_connections 2023-11-17T05:32:10.239 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:10.239+0000 7fcec050c700 1 -- 172.21.15.66:0/1597146305 wait complete. 2023-11-17T05:32:10.241 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:10.239+0000 7fcec050c700 1 Processor -- start 2023-11-17T05:32:10.241 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:10.239+0000 7fcec050c700 1 -- start start 2023-11-17T05:32:10.241 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:10.239+0000 7fcec050c700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fceb80f5410 0x7fceb8109450 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:32:10.241 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:10.239+0000 7fcec050c700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fceb8060220 con 0x7fceb80f5410 2023-11-17T05:32:10.241 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:10.240+0000 7fcebe2a8700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fceb80f5410 0x7fceb8109450 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:32:10.241 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:10.240+0000 7fcebe2a8700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fceb80f5410 0x7fceb8109450 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.66:37814/0 (socket says 172.21.15.66:37814) 2023-11-17T05:32:10.241 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:10.240+0000 7fcebe2a8700 1 -- 172.21.15.66:0/3921473917 learned_addr learned my addr 172.21.15.66:0/3921473917 (peer_addr_for_me v2:172.21.15.66:0/0) 2023-11-17T05:32:10.241 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:10.240+0000 7fcebe2a8700 1 -- 172.21.15.66:0/3921473917 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fceb4005040 con 0x7fceb80f5410 2023-11-17T05:32:10.241 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:10.240+0000 7fcebe2a8700 1 --2- 172.21.15.66:0/3921473917 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fceb80f5410 0x7fceb8109450 secure :-1 s=READY pgs=358 cs=0 l=1 rev1=1 rx=0x7fceb4013510 tx=0x7fceb400b040).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:32:10.242 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:10.240+0000 7fceaaffd700 1 -- 172.21.15.66:0/3921473917 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7fceb4009320 con 0x7fceb80f5410 2023-11-17T05:32:10.242 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:10.240+0000 7fceaaffd700 1 -- 172.21.15.66:0/3921473917 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(25 keys) v1 ==== 1074+0+0 (secure 0 0 0) 0x7fceb4013f00 con 0x7fceb80f5410 2023-11-17T05:32:10.242 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:10.240+0000 7fcec050c700 1 -- 172.21.15.66:0/3921473917 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fceb8060420 con 0x7fceb80f5410 2023-11-17T05:32:10.242 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:10.240+0000 7fceaaffd700 1 -- 172.21.15.66:0/3921473917 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7fceb4014d90 con 0x7fceb80f5410 2023-11-17T05:32:10.242 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:10.240+0000 7fcec050c700 1 -- 172.21.15.66:0/3921473917 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fceb8109b00 con 0x7fceb80f5410 2023-11-17T05:32:10.243 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:10.241+0000 7fceaaffd700 1 -- 172.21.15.66:0/3921473917 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 17) v1 ==== 36269+0+0 (secure 0 0 0) 0x7fceb4014420 con 0x7fceb80f5410 2023-11-17T05:32:10.243 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:10.241+0000 7fcec050c700 1 -- 172.21.15.66:0/3921473917 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fceb8059100 con 0x7fceb80f5410 2023-11-17T05:32:10.243 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:10.241+0000 7fceaaffd700 1 --2- 172.21.15.66:0/3921473917 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fcea402ffb0 0x7fcea4032470 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:32:10.243 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:10.241+0000 7fceaaffd700 1 -- 172.21.15.66:0/3921473917 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(5..5 src has 1..5) v4 ==== 1270+0+0 (secure 0 0 0) 0x7fceb4040fa0 con 0x7fceb80f5410 2023-11-17T05:32:10.243 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:10.242+0000 7fcebdaa7700 1 --2- 172.21.15.66:0/3921473917 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fcea402ffb0 0x7fcea4032470 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:32:10.243 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:10.242+0000 7fcebdaa7700 1 --2- 172.21.15.66:0/3921473917 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fcea402ffb0 0x7fcea4032470 secure :-1 s=READY pgs=144 cs=0 l=1 rev1=1 rx=0x7fceac0096a0 tx=0x7fceac006b40).ready entity=mgr.14164 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:32:10.247 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:10.247+0000 7fceaaffd700 1 -- 172.21.15.66:0/3921473917 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+152212 (secure 0 0 0) 0x7fceb406d2b0 con 0x7fceb80f5410 2023-11-17T05:32:10.472 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:10.472+0000 7fcec050c700 1 -- 172.21.15.66:0/3921473917 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "mon dump", "format": "json"} v 0) v1 -- 0x7fceb8045ad0 con 0x7fceb80f5410 2023-11-17T05:32:10.472 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:10.472+0000 7fceaaffd700 1 -- 172.21.15.66:0/3921473917 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mon_command_ack([{"prefix": "mon dump", "format": "json"}]=0 dumped monmap epoch 1 v1) v1 ==== 95+0+690 (secure 0 0 0) 0x7fceb40095f0 con 0x7fceb80f5410 2023-11-17T05:32:10.473 INFO:teuthology.orchestra.run.smithi066.stdout: 2023-11-17T05:32:10.473 INFO:teuthology.orchestra.run.smithi066.stdout:{"epoch":1,"fsid":"ed6470e2-8508-11ee-8dbe-212e2dc638e7","modified":"2023-11-17T05:20:13.934038Z","created":"2023-11-17T05:20:13.934038Z","min_mon_release":16,"min_mon_release_name":"pacific","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging"],"optional":[]},"mons":[{"rank":0,"name":"smithi040","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:3300","nonce":0},{"type":"v1","addr":"172.21.15.40:6789","nonce":0}]},"addr":"172.21.15.40:6789/0","public_addr":"172.21.15.40:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2023-11-17T05:32:10.475 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:10.474+0000 7fcec050c700 1 -- 172.21.15.66:0/3921473917 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fcea402ffb0 msgr2=0x7fcea4032470 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:32:10.476 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:10.474+0000 7fcec050c700 1 --2- 172.21.15.66:0/3921473917 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fcea402ffb0 0x7fcea4032470 secure :-1 s=READY pgs=144 cs=0 l=1 rev1=1 rx=0x7fceac0096a0 tx=0x7fceac006b40).stop 2023-11-17T05:32:10.476 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:10.474+0000 7fcec050c700 1 -- 172.21.15.66:0/3921473917 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fceb80f5410 msgr2=0x7fceb8109450 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:32:10.476 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:10.474+0000 7fcec050c700 1 --2- 172.21.15.66:0/3921473917 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fceb80f5410 0x7fceb8109450 secure :-1 s=READY pgs=358 cs=0 l=1 rev1=1 rx=0x7fceb4013510 tx=0x7fceb400b040).stop 2023-11-17T05:32:10.476 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:10.474+0000 7fcec050c700 1 -- 172.21.15.66:0/3921473917 shutdown_connections 2023-11-17T05:32:10.476 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:10.474+0000 7fcec050c700 1 --2- 172.21.15.66:0/3921473917 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fcea402ffb0 0x7fcea4032470 unknown :-1 s=CLOSED pgs=144 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:32:10.476 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:10.474+0000 7fcec050c700 1 --2- 172.21.15.66:0/3921473917 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fceb80f5410 0x7fceb8109450 unknown :-1 s=CLOSED pgs=358 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:32:10.476 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:10.474+0000 7fcec050c700 1 -- 172.21.15.66:0/3921473917 >> 172.21.15.66:0/3921473917 conn(0x7fceb80f0b70 msgr2=0x7fceb80f1660 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:32:10.476 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:10.474+0000 7fcec050c700 1 -- 172.21.15.66:0/3921473917 shutdown_connections 2023-11-17T05:32:10.477 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:10.474+0000 7fcec050c700 1 -- 172.21.15.66:0/3921473917 wait complete. 2023-11-17T05:32:10.477 INFO:teuthology.orchestra.run.smithi066.stderr:dumped monmap epoch 1 2023-11-17T05:32:10.989 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:32:10 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:32:09.522497+0000 mgr.smithi040.ogcprl (mgr.14164) 340 : cluster [DBG] pgmap v324: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:32:10.989 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:32:10 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:32:10.472890+0000 mon.smithi040 (mon.0) 289 : audit [DBG] from='client.? 172.21.15.66:0/3921473917' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-11-17T05:32:11.866 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2023-11-17T05:32:11.866 DEBUG:teuthology.orchestra.run.smithi066:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.4 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ed6470e2-8508-11ee-8dbe-212e2dc638e7 -- ceph mon dump -f json 2023-11-17T05:32:11.990 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:32:11 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:32:11.319723+0000 mon.smithi040 (mon.0) 290 : audit [INF] from='mgr.14164 172.21.15.40:0/2203032451' entity='mgr.smithi040.ogcprl' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi040.ogcprl/mirror_snapshot_schedule"}]: dispatch 2023-11-17T05:32:11.990 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:32:11 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:32:11.319965+0000 mon.smithi040 (mon.0) 291 : audit [INF] from='mgr.14164 172.21.15.40:0/2203032451' entity='mgr.smithi040.ogcprl' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi040.ogcprl/trash_purge_schedule"}]: dispatch 2023-11-17T05:32:12.988 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:32:12 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:32:11.522786+0000 mgr.smithi040.ogcprl (mgr.14164) 341 : cluster [DBG] pgmap v325: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:32:13.051 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:13.049+0000 7fec65358700 1 -- 172.21.15.66:0/2358045822 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fec600f6510 msgr2=0x7fec600f6930 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:32:13.051 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:13.049+0000 7fec65358700 1 --2- 172.21.15.66:0/2358045822 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fec600f6510 0x7fec600f6930 secure :-1 s=READY pgs=359 cs=0 l=1 rev1=1 rx=0x7fec54009fb0 tx=0x7fec54009b30).stop 2023-11-17T05:32:13.051 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:13.049+0000 7fec65358700 1 -- 172.21.15.66:0/2358045822 shutdown_connections 2023-11-17T05:32:13.051 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:13.049+0000 7fec65358700 1 --2- 172.21.15.66:0/2358045822 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fec600f6510 0x7fec600f6930 unknown :-1 s=CLOSED pgs=359 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:32:13.051 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:13.049+0000 7fec65358700 1 -- 172.21.15.66:0/2358045822 >> 172.21.15.66:0/2358045822 conn(0x7fec600f1c70 msgr2=0x7fec600f4090 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:32:13.051 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:13.049+0000 7fec65358700 1 -- 172.21.15.66:0/2358045822 shutdown_connections 2023-11-17T05:32:13.051 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:13.049+0000 7fec65358700 1 -- 172.21.15.66:0/2358045822 wait complete. 2023-11-17T05:32:13.051 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:13.050+0000 7fec65358700 1 Processor -- start 2023-11-17T05:32:13.051 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:13.050+0000 7fec65358700 1 -- start start 2023-11-17T05:32:13.052 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:13.050+0000 7fec65358700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fec600f6510 0x7fec601066d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:32:13.052 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:13.050+0000 7fec65358700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fec60106bc0 con 0x7fec600f6510 2023-11-17T05:32:13.052 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:13.050+0000 7fec5effd700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fec600f6510 0x7fec601066d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:32:13.052 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:13.050+0000 7fec5effd700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fec600f6510 0x7fec601066d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.66:54588/0 (socket says 172.21.15.66:54588) 2023-11-17T05:32:13.052 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:13.050+0000 7fec5effd700 1 -- 172.21.15.66:0/1704510848 learned_addr learned my addr 172.21.15.66:0/1704510848 (peer_addr_for_me v2:172.21.15.66:0/0) 2023-11-17T05:32:13.052 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:13.051+0000 7fec5effd700 1 -- 172.21.15.66:0/1704510848 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fec54005040 con 0x7fec600f6510 2023-11-17T05:32:13.052 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:13.051+0000 7fec5effd700 1 --2- 172.21.15.66:0/1704510848 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fec600f6510 0x7fec601066d0 secure :-1 s=READY pgs=360 cs=0 l=1 rev1=1 rx=0x7fec54013510 tx=0x7fec5400b040).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:32:13.053 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:13.051+0000 7fec477fe700 1 -- 172.21.15.66:0/1704510848 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7fec54009320 con 0x7fec600f6510 2023-11-17T05:32:13.053 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:13.051+0000 7fec65358700 1 -- 172.21.15.66:0/1704510848 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fec60106dc0 con 0x7fec600f6510 2023-11-17T05:32:13.053 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:13.051+0000 7fec477fe700 1 -- 172.21.15.66:0/1704510848 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(25 keys) v1 ==== 1074+0+0 (secure 0 0 0) 0x7fec54013f00 con 0x7fec600f6510 2023-11-17T05:32:13.053 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:13.051+0000 7fec477fe700 1 -- 172.21.15.66:0/1704510848 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7fec54014d90 con 0x7fec600f6510 2023-11-17T05:32:13.054 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:13.051+0000 7fec65358700 1 -- 172.21.15.66:0/1704510848 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fec60107250 con 0x7fec600f6510 2023-11-17T05:32:13.054 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:13.052+0000 7fec477fe700 1 -- 172.21.15.66:0/1704510848 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 17) v1 ==== 36269+0+0 (secure 0 0 0) 0x7fec540263a0 con 0x7fec600f6510 2023-11-17T05:32:13.054 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:13.052+0000 7fec477fe700 1 --2- 172.21.15.66:0/1704510848 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fec4802ff60 0x7fec48032420 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:32:13.054 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:13.052+0000 7fec477fe700 1 -- 172.21.15.66:0/1704510848 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(5..5 src has 1..5) v4 ==== 1270+0+0 (secure 0 0 0) 0x7fec54041020 con 0x7fec600f6510 2023-11-17T05:32:13.054 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:13.052+0000 7fec65358700 1 -- 172.21.15.66:0/1704510848 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fec60045ad0 con 0x7fec600f6510 2023-11-17T05:32:13.054 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:13.052+0000 7fec5e7fc700 1 --2- 172.21.15.66:0/1704510848 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fec4802ff60 0x7fec48032420 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:32:13.054 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:13.053+0000 7fec5e7fc700 1 --2- 172.21.15.66:0/1704510848 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fec4802ff60 0x7fec48032420 secure :-1 s=READY pgs=145 cs=0 l=1 rev1=1 rx=0x7fec500096a0 tx=0x7fec50006b40).ready entity=mgr.14164 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:32:13.057 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:13.057+0000 7fec477fe700 1 -- 172.21.15.66:0/1704510848 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+152212 (secure 0 0 0) 0x7fec54012920 con 0x7fec600f6510 2023-11-17T05:32:13.278 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:13.278+0000 7fec65358700 1 -- 172.21.15.66:0/1704510848 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "mon dump", "format": "json"} v 0) v1 -- 0x7fec60107fa0 con 0x7fec600f6510 2023-11-17T05:32:13.279 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:13.279+0000 7fec477fe700 1 -- 172.21.15.66:0/1704510848 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mon_command_ack([{"prefix": "mon dump", "format": "json"}]=0 dumped monmap epoch 1 v1) v1 ==== 95+0+690 (secure 0 0 0) 0x7fec5402b020 con 0x7fec600f6510 2023-11-17T05:32:13.279 INFO:teuthology.orchestra.run.smithi066.stdout: 2023-11-17T05:32:13.279 INFO:teuthology.orchestra.run.smithi066.stdout:{"epoch":1,"fsid":"ed6470e2-8508-11ee-8dbe-212e2dc638e7","modified":"2023-11-17T05:20:13.934038Z","created":"2023-11-17T05:20:13.934038Z","min_mon_release":16,"min_mon_release_name":"pacific","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging"],"optional":[]},"mons":[{"rank":0,"name":"smithi040","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:3300","nonce":0},{"type":"v1","addr":"172.21.15.40:6789","nonce":0}]},"addr":"172.21.15.40:6789/0","public_addr":"172.21.15.40:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2023-11-17T05:32:13.281 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:13.280+0000 7fec65358700 1 -- 172.21.15.66:0/1704510848 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fec4802ff60 msgr2=0x7fec48032420 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:32:13.281 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:13.280+0000 7fec65358700 1 --2- 172.21.15.66:0/1704510848 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fec4802ff60 0x7fec48032420 secure :-1 s=READY pgs=145 cs=0 l=1 rev1=1 rx=0x7fec500096a0 tx=0x7fec50006b40).stop 2023-11-17T05:32:13.281 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:13.280+0000 7fec65358700 1 -- 172.21.15.66:0/1704510848 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fec600f6510 msgr2=0x7fec601066d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:32:13.281 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:13.280+0000 7fec65358700 1 --2- 172.21.15.66:0/1704510848 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fec600f6510 0x7fec601066d0 secure :-1 s=READY pgs=360 cs=0 l=1 rev1=1 rx=0x7fec54013510 tx=0x7fec5400b040).stop 2023-11-17T05:32:13.281 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:13.280+0000 7fec65358700 1 -- 172.21.15.66:0/1704510848 shutdown_connections 2023-11-17T05:32:13.281 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:13.280+0000 7fec65358700 1 --2- 172.21.15.66:0/1704510848 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fec4802ff60 0x7fec48032420 unknown :-1 s=CLOSED pgs=145 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:32:13.281 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:13.280+0000 7fec65358700 1 --2- 172.21.15.66:0/1704510848 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fec600f6510 0x7fec601066d0 unknown :-1 s=CLOSED pgs=360 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:32:13.282 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:13.280+0000 7fec65358700 1 -- 172.21.15.66:0/1704510848 >> 172.21.15.66:0/1704510848 conn(0x7fec600f1c70 msgr2=0x7fec600f2710 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:32:13.282 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:13.280+0000 7fec65358700 1 -- 172.21.15.66:0/1704510848 shutdown_connections 2023-11-17T05:32:13.290 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:13.281+0000 7fec65358700 1 -- 172.21.15.66:0/1704510848 wait complete. 2023-11-17T05:32:13.291 INFO:teuthology.orchestra.run.smithi066.stderr:dumped monmap epoch 1 2023-11-17T05:32:13.992 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:32:13 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:32:13.279428+0000 mon.smithi040 (mon.0) 292 : audit [DBG] from='client.? 172.21.15.66:0/1704510848' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-11-17T05:32:14.656 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2023-11-17T05:32:14.656 DEBUG:teuthology.orchestra.run.smithi066:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.4 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ed6470e2-8508-11ee-8dbe-212e2dc638e7 -- ceph mon dump -f json 2023-11-17T05:32:14.989 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:32:14 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:32:13.523161+0000 mgr.smithi040.ogcprl (mgr.14164) 342 : cluster [DBG] pgmap v326: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:32:15.899 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:15.897+0000 7f33b6ba0700 1 -- 172.21.15.66:0/1949256241 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f33b0060180 msgr2=0x7f33b00f8f80 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:32:15.899 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:15.897+0000 7f33b6ba0700 1 --2- 172.21.15.66:0/1949256241 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f33b0060180 0x7f33b00f8f80 secure :-1 s=READY pgs=361 cs=0 l=1 rev1=1 rx=0x7f33ac009fb0 tx=0x7f33ac009b30).stop 2023-11-17T05:32:15.899 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:15.897+0000 7f33b6ba0700 1 -- 172.21.15.66:0/1949256241 shutdown_connections 2023-11-17T05:32:15.900 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:15.897+0000 7f33b6ba0700 1 --2- 172.21.15.66:0/1949256241 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f33b0060180 0x7f33b00f8f80 unknown :-1 s=CLOSED pgs=361 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:32:15.900 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:15.897+0000 7f33b6ba0700 1 -- 172.21.15.66:0/1949256241 >> 172.21.15.66:0/1949256241 conn(0x7f33b00f09a0 msgr2=0x7f33b00f2dc0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:32:15.900 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:15.897+0000 7f33b6ba0700 1 -- 172.21.15.66:0/1949256241 shutdown_connections 2023-11-17T05:32:15.900 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:15.897+0000 7f33b6ba0700 1 -- 172.21.15.66:0/1949256241 wait complete. 2023-11-17T05:32:15.900 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:15.898+0000 7f33b6ba0700 1 Processor -- start 2023-11-17T05:32:15.900 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:15.898+0000 7f33b6ba0700 1 -- start start 2023-11-17T05:32:15.900 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:15.898+0000 7f33b6ba0700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f33b0060180 0x7f33b0106500 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:32:15.900 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:15.898+0000 7f33b6ba0700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f33b01069f0 con 0x7f33b0060180 2023-11-17T05:32:15.900 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:15.898+0000 7f33b5b9e700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f33b0060180 0x7f33b0106500 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:32:15.900 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:15.898+0000 7f33b5b9e700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f33b0060180 0x7f33b0106500 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.66:54596/0 (socket says 172.21.15.66:54596) 2023-11-17T05:32:15.900 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:15.898+0000 7f33b5b9e700 1 -- 172.21.15.66:0/1095804421 learned_addr learned my addr 172.21.15.66:0/1095804421 (peer_addr_for_me v2:172.21.15.66:0/0) 2023-11-17T05:32:15.900 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:15.899+0000 7f33b5b9e700 1 -- 172.21.15.66:0/1095804421 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f33ac005040 con 0x7f33b0060180 2023-11-17T05:32:15.901 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:15.899+0000 7f33b5b9e700 1 --2- 172.21.15.66:0/1095804421 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f33b0060180 0x7f33b0106500 secure :-1 s=READY pgs=362 cs=0 l=1 rev1=1 rx=0x7f33ac013510 tx=0x7f33ac00b040).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:32:15.901 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:15.899+0000 7f33a27fc700 1 -- 172.21.15.66:0/1095804421 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f33ac009320 con 0x7f33b0060180 2023-11-17T05:32:15.901 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:15.899+0000 7f33a27fc700 1 -- 172.21.15.66:0/1095804421 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(25 keys) v1 ==== 1074+0+0 (secure 0 0 0) 0x7f33ac013f00 con 0x7f33b0060180 2023-11-17T05:32:15.901 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:15.899+0000 7f33b6ba0700 1 -- 172.21.15.66:0/1095804421 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f33b0106bf0 con 0x7f33b0060180 2023-11-17T05:32:15.901 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:15.899+0000 7f33a27fc700 1 -- 172.21.15.66:0/1095804421 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f33ac014d90 con 0x7f33b0060180 2023-11-17T05:32:15.901 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:15.899+0000 7f33b6ba0700 1 -- 172.21.15.66:0/1095804421 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f33b0107080 con 0x7f33b0060180 2023-11-17T05:32:15.902 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:15.900+0000 7f33a27fc700 1 -- 172.21.15.66:0/1095804421 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 17) v1 ==== 36269+0+0 (secure 0 0 0) 0x7f33ac0263a0 con 0x7f33b0060180 2023-11-17T05:32:15.902 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:15.900+0000 7f33a27fc700 1 --2- 172.21.15.66:0/1095804421 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f339c02ff60 0x7f339c032420 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:32:15.902 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:15.900+0000 7f33b6ba0700 1 -- 172.21.15.66:0/1095804421 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f33b0045ad0 con 0x7f33b0060180 2023-11-17T05:32:15.902 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:15.900+0000 7f33a27fc700 1 -- 172.21.15.66:0/1095804421 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(5..5 src has 1..5) v4 ==== 1270+0+0 (secure 0 0 0) 0x7f33ac0410a0 con 0x7f33b0060180 2023-11-17T05:32:15.902 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:15.900+0000 7f33b539d700 1 --2- 172.21.15.66:0/1095804421 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f339c02ff60 0x7f339c032420 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:32:15.902 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:15.900+0000 7f33b539d700 1 --2- 172.21.15.66:0/1095804421 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f339c02ff60 0x7f339c032420 secure :-1 s=READY pgs=146 cs=0 l=1 rev1=1 rx=0x7f33a40096a0 tx=0x7f33a4006b40).ready entity=mgr.14164 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:32:15.905 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:15.905+0000 7f33a27fc700 1 -- 172.21.15.66:0/1095804421 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+152212 (secure 0 0 0) 0x7f33ac012920 con 0x7f33b0060180 2023-11-17T05:32:15.989 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:32:15 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:32:15.523487+0000 mgr.smithi040.ogcprl (mgr.14164) 343 : cluster [DBG] pgmap v327: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:32:16.123 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:16.123+0000 7f33b6ba0700 1 -- 172.21.15.66:0/1095804421 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "mon dump", "format": "json"} v 0) v1 -- 0x7f33b0107f30 con 0x7f33b0060180 2023-11-17T05:32:16.124 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:16.124+0000 7f33a27fc700 1 -- 172.21.15.66:0/1095804421 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mon_command_ack([{"prefix": "mon dump", "format": "json"}]=0 dumped monmap epoch 1 v1) v1 ==== 95+0+690 (secure 0 0 0) 0x7f33ac02c090 con 0x7f33b0060180 2023-11-17T05:32:16.124 INFO:teuthology.orchestra.run.smithi066.stdout: 2023-11-17T05:32:16.124 INFO:teuthology.orchestra.run.smithi066.stdout:{"epoch":1,"fsid":"ed6470e2-8508-11ee-8dbe-212e2dc638e7","modified":"2023-11-17T05:20:13.934038Z","created":"2023-11-17T05:20:13.934038Z","min_mon_release":16,"min_mon_release_name":"pacific","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging"],"optional":[]},"mons":[{"rank":0,"name":"smithi040","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:3300","nonce":0},{"type":"v1","addr":"172.21.15.40:6789","nonce":0}]},"addr":"172.21.15.40:6789/0","public_addr":"172.21.15.40:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2023-11-17T05:32:16.126 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:16.125+0000 7f33b6ba0700 1 -- 172.21.15.66:0/1095804421 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f339c02ff60 msgr2=0x7f339c032420 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:32:16.126 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:16.125+0000 7f33b6ba0700 1 --2- 172.21.15.66:0/1095804421 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f339c02ff60 0x7f339c032420 secure :-1 s=READY pgs=146 cs=0 l=1 rev1=1 rx=0x7f33a40096a0 tx=0x7f33a4006b40).stop 2023-11-17T05:32:16.126 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:16.125+0000 7f33b6ba0700 1 -- 172.21.15.66:0/1095804421 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f33b0060180 msgr2=0x7f33b0106500 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:32:16.126 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:16.125+0000 7f33b6ba0700 1 --2- 172.21.15.66:0/1095804421 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f33b0060180 0x7f33b0106500 secure :-1 s=READY pgs=362 cs=0 l=1 rev1=1 rx=0x7f33ac013510 tx=0x7f33ac00b040).stop 2023-11-17T05:32:16.126 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:16.125+0000 7f33b6ba0700 1 -- 172.21.15.66:0/1095804421 shutdown_connections 2023-11-17T05:32:16.126 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:16.125+0000 7f33b6ba0700 1 --2- 172.21.15.66:0/1095804421 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f339c02ff60 0x7f339c032420 unknown :-1 s=CLOSED pgs=146 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:32:16.126 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:16.125+0000 7f33b6ba0700 1 --2- 172.21.15.66:0/1095804421 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f33b0060180 0x7f33b0106500 unknown :-1 s=CLOSED pgs=362 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:32:16.126 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:16.125+0000 7f33b6ba0700 1 -- 172.21.15.66:0/1095804421 >> 172.21.15.66:0/1095804421 conn(0x7f33b00f09a0 msgr2=0x7f33b00f1460 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:32:16.126 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:16.125+0000 7f33b6ba0700 1 -- 172.21.15.66:0/1095804421 shutdown_connections 2023-11-17T05:32:16.126 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:16.125+0000 7f33b6ba0700 1 -- 172.21.15.66:0/1095804421 wait complete. 2023-11-17T05:32:16.127 INFO:teuthology.orchestra.run.smithi066.stderr:dumped monmap epoch 1 2023-11-17T05:32:16.989 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:32:16 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:32:16.124351+0000 mon.smithi040 (mon.0) 293 : audit [DBG] from='client.? 172.21.15.66:0/1095804421' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-11-17T05:32:17.509 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2023-11-17T05:32:17.510 DEBUG:teuthology.orchestra.run.smithi066:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.4 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ed6470e2-8508-11ee-8dbe-212e2dc638e7 -- ceph mon dump -f json 2023-11-17T05:32:17.988 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:32:17 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:32:17.523811+0000 mgr.smithi040.ogcprl (mgr.14164) 344 : cluster [DBG] pgmap v328: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:32:18.757 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:18.756+0000 7fb2f30c8700 1 -- 172.21.15.66:0/1560203464 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fb2ec0601e0 msgr2=0x7fb2ec0f9190 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:32:18.757 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:18.756+0000 7fb2f30c8700 1 --2- 172.21.15.66:0/1560203464 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fb2ec0601e0 0x7fb2ec0f9190 secure :-1 s=READY pgs=363 cs=0 l=1 rev1=1 rx=0x7fb2e0009fb0 tx=0x7fb2e0009b30).stop 2023-11-17T05:32:18.758 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:18.756+0000 7fb2f30c8700 1 -- 172.21.15.66:0/1560203464 shutdown_connections 2023-11-17T05:32:18.758 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:18.756+0000 7fb2f30c8700 1 --2- 172.21.15.66:0/1560203464 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fb2ec0601e0 0x7fb2ec0f9190 unknown :-1 s=CLOSED pgs=363 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:32:18.758 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:18.756+0000 7fb2f30c8700 1 -- 172.21.15.66:0/1560203464 >> 172.21.15.66:0/1560203464 conn(0x7fb2ec0f0b70 msgr2=0x7fb2ec0f2f90 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:32:18.758 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:18.756+0000 7fb2f30c8700 1 -- 172.21.15.66:0/1560203464 shutdown_connections 2023-11-17T05:32:18.758 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:18.756+0000 7fb2f30c8700 1 -- 172.21.15.66:0/1560203464 wait complete. 2023-11-17T05:32:18.758 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:18.757+0000 7fb2f30c8700 1 Processor -- start 2023-11-17T05:32:18.758 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:18.757+0000 7fb2f30c8700 1 -- start start 2023-11-17T05:32:18.758 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:18.757+0000 7fb2f30c8700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fb2ec0601e0 0x7fb2ec1066b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:32:18.758 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:18.757+0000 7fb2f30c8700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fb2ec106ba0 con 0x7fb2ec0601e0 2023-11-17T05:32:18.760 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:18.757+0000 7fb2f0e64700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fb2ec0601e0 0x7fb2ec1066b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:32:18.760 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:18.757+0000 7fb2f0e64700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fb2ec0601e0 0x7fb2ec1066b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.66:54616/0 (socket says 172.21.15.66:54616) 2023-11-17T05:32:18.760 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:18.757+0000 7fb2f0e64700 1 -- 172.21.15.66:0/1179985798 learned_addr learned my addr 172.21.15.66:0/1179985798 (peer_addr_for_me v2:172.21.15.66:0/0) 2023-11-17T05:32:18.760 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:18.758+0000 7fb2f0e64700 1 -- 172.21.15.66:0/1179985798 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fb2e0005040 con 0x7fb2ec0601e0 2023-11-17T05:32:18.760 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:18.758+0000 7fb2f0e64700 1 --2- 172.21.15.66:0/1179985798 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fb2ec0601e0 0x7fb2ec1066b0 secure :-1 s=READY pgs=364 cs=0 l=1 rev1=1 rx=0x7fb2e0004dd0 tx=0x7fb2e0009b30).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:32:18.760 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:18.758+0000 7fb2e97fa700 1 -- 172.21.15.66:0/1179985798 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7fb2e0018070 con 0x7fb2ec0601e0 2023-11-17T05:32:18.760 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:18.758+0000 7fb2e97fa700 1 -- 172.21.15.66:0/1179985798 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(25 keys) v1 ==== 1074+0+0 (secure 0 0 0) 0x7fb2e0013410 con 0x7fb2ec0601e0 2023-11-17T05:32:18.761 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:18.758+0000 7fb2f30c8700 1 -- 172.21.15.66:0/1179985798 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fb2ec106da0 con 0x7fb2ec0601e0 2023-11-17T05:32:18.761 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:18.758+0000 7fb2e97fa700 1 -- 172.21.15.66:0/1179985798 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7fb2e001c3c0 con 0x7fb2ec0601e0 2023-11-17T05:32:18.761 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:18.758+0000 7fb2f30c8700 1 -- 172.21.15.66:0/1179985798 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fb2ec107230 con 0x7fb2ec0601e0 2023-11-17T05:32:18.761 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:18.759+0000 7fb2e97fa700 1 -- 172.21.15.66:0/1179985798 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 17) v1 ==== 36269+0+0 (secure 0 0 0) 0x7fb2e001d420 con 0x7fb2ec0601e0 2023-11-17T05:32:18.761 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:18.759+0000 7fb2f30c8700 1 -- 172.21.15.66:0/1179985798 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fb2ec045ad0 con 0x7fb2ec0601e0 2023-11-17T05:32:18.761 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:18.759+0000 7fb2e97fa700 1 --2- 172.21.15.66:0/1179985798 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fb2dc02ff60 0x7fb2dc032420 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:32:18.761 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:18.759+0000 7fb2e97fa700 1 -- 172.21.15.66:0/1179985798 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(5..5 src has 1..5) v4 ==== 1270+0+0 (secure 0 0 0) 0x7fb2e0041510 con 0x7fb2ec0601e0 2023-11-17T05:32:18.761 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:18.759+0000 7fb2ebfff700 1 --2- 172.21.15.66:0/1179985798 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fb2dc02ff60 0x7fb2dc032420 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:32:18.761 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:18.760+0000 7fb2ebfff700 1 --2- 172.21.15.66:0/1179985798 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fb2dc02ff60 0x7fb2dc032420 secure :-1 s=READY pgs=147 cs=0 l=1 rev1=1 rx=0x7fb2d80096a0 tx=0x7fb2d8006b40).ready entity=mgr.14164 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:32:18.764 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:18.764+0000 7fb2e97fa700 1 -- 172.21.15.66:0/1179985798 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+152212 (secure 0 0 0) 0x7fb2e001d820 con 0x7fb2ec0601e0 2023-11-17T05:32:18.988 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:18.987+0000 7fb2f30c8700 1 -- 172.21.15.66:0/1179985798 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "mon dump", "format": "json"} v 0) v1 -- 0x7fb2ec1080e0 con 0x7fb2ec0601e0 2023-11-17T05:32:18.988 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:18.988+0000 7fb2e97fa700 1 -- 172.21.15.66:0/1179985798 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mon_command_ack([{"prefix": "mon dump", "format": "json"}]=0 dumped monmap epoch 1 v1) v1 ==== 95+0+690 (secure 0 0 0) 0x7fb2e001b370 con 0x7fb2ec0601e0 2023-11-17T05:32:18.989 INFO:teuthology.orchestra.run.smithi066.stdout: 2023-11-17T05:32:18.989 INFO:teuthology.orchestra.run.smithi066.stdout:{"epoch":1,"fsid":"ed6470e2-8508-11ee-8dbe-212e2dc638e7","modified":"2023-11-17T05:20:13.934038Z","created":"2023-11-17T05:20:13.934038Z","min_mon_release":16,"min_mon_release_name":"pacific","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging"],"optional":[]},"mons":[{"rank":0,"name":"smithi040","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:3300","nonce":0},{"type":"v1","addr":"172.21.15.40:6789","nonce":0}]},"addr":"172.21.15.40:6789/0","public_addr":"172.21.15.40:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2023-11-17T05:32:18.991 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:18.990+0000 7fb2f30c8700 1 -- 172.21.15.66:0/1179985798 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fb2dc02ff60 msgr2=0x7fb2dc032420 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:32:18.991 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:18.990+0000 7fb2f30c8700 1 --2- 172.21.15.66:0/1179985798 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fb2dc02ff60 0x7fb2dc032420 secure :-1 s=READY pgs=147 cs=0 l=1 rev1=1 rx=0x7fb2d80096a0 tx=0x7fb2d8006b40).stop 2023-11-17T05:32:18.991 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:18.990+0000 7fb2f30c8700 1 -- 172.21.15.66:0/1179985798 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fb2ec0601e0 msgr2=0x7fb2ec1066b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:32:18.991 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:18.990+0000 7fb2f30c8700 1 --2- 172.21.15.66:0/1179985798 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fb2ec0601e0 0x7fb2ec1066b0 secure :-1 s=READY pgs=364 cs=0 l=1 rev1=1 rx=0x7fb2e0004dd0 tx=0x7fb2e0009b30).stop 2023-11-17T05:32:18.991 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:18.990+0000 7fb2f30c8700 1 -- 172.21.15.66:0/1179985798 shutdown_connections 2023-11-17T05:32:18.991 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:18.990+0000 7fb2f30c8700 1 --2- 172.21.15.66:0/1179985798 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fb2dc02ff60 0x7fb2dc032420 unknown :-1 s=CLOSED pgs=147 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:32:18.991 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:18.990+0000 7fb2f30c8700 1 --2- 172.21.15.66:0/1179985798 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fb2ec0601e0 0x7fb2ec1066b0 unknown :-1 s=CLOSED pgs=364 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:32:18.991 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:18.990+0000 7fb2f30c8700 1 -- 172.21.15.66:0/1179985798 >> 172.21.15.66:0/1179985798 conn(0x7fb2ec0f0b70 msgr2=0x7fb2ec0f1650 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:32:18.991 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:18.990+0000 7fb2f30c8700 1 -- 172.21.15.66:0/1179985798 shutdown_connections 2023-11-17T05:32:18.991 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:18.990+0000 7fb2f30c8700 1 -- 172.21.15.66:0/1179985798 wait complete. 2023-11-17T05:32:18.992 INFO:teuthology.orchestra.run.smithi066.stderr:dumped monmap epoch 1 2023-11-17T05:32:19.489 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:32:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:32:18.988810+0000 mon.smithi040 (mon.0) 294 : audit [DBG] from='client.? 172.21.15.66:0/1179985798' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-11-17T05:32:20.410 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2023-11-17T05:32:20.410 DEBUG:teuthology.orchestra.run.smithi066:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.4 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ed6470e2-8508-11ee-8dbe-212e2dc638e7 -- ceph mon dump -f json 2023-11-17T05:32:20.989 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:32:20 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:32:19.524139+0000 mgr.smithi040.ogcprl (mgr.14164) 345 : cluster [DBG] pgmap v329: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:32:21.553 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:21.553+0000 7fcb35a49700 1 -- 172.21.15.66:0/3040939303 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fcb300f6510 msgr2=0x7fcb300f6930 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:32:21.553 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:21.553+0000 7fcb35a49700 1 --2- 172.21.15.66:0/3040939303 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fcb300f6510 0x7fcb300f6930 secure :-1 s=READY pgs=365 cs=0 l=1 rev1=1 rx=0x7fcb20009fb0 tx=0x7fcb20009b30).stop 2023-11-17T05:32:21.553 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:21.553+0000 7fcb35a49700 1 -- 172.21.15.66:0/3040939303 shutdown_connections 2023-11-17T05:32:21.553 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:21.553+0000 7fcb35a49700 1 --2- 172.21.15.66:0/3040939303 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fcb300f6510 0x7fcb300f6930 unknown :-1 s=CLOSED pgs=365 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:32:21.554 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:21.553+0000 7fcb35a49700 1 -- 172.21.15.66:0/3040939303 >> 172.21.15.66:0/3040939303 conn(0x7fcb300f1c70 msgr2=0x7fcb300f4090 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:32:21.554 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:21.553+0000 7fcb35a49700 1 -- 172.21.15.66:0/3040939303 shutdown_connections 2023-11-17T05:32:21.554 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:21.553+0000 7fcb35a49700 1 -- 172.21.15.66:0/3040939303 wait complete. 2023-11-17T05:32:21.555 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:21.553+0000 7fcb35a49700 1 Processor -- start 2023-11-17T05:32:21.555 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:21.553+0000 7fcb35a49700 1 -- start start 2023-11-17T05:32:21.555 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:21.554+0000 7fcb35a49700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fcb300f6510 0x7fcb30108900 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:32:21.555 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:21.554+0000 7fcb35a49700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fcb30108df0 con 0x7fcb300f6510 2023-11-17T05:32:21.555 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:21.554+0000 7fcb2effd700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fcb300f6510 0x7fcb30108900 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:32:21.555 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:21.554+0000 7fcb2effd700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fcb300f6510 0x7fcb30108900 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.66:54638/0 (socket says 172.21.15.66:54638) 2023-11-17T05:32:21.556 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:21.554+0000 7fcb2effd700 1 -- 172.21.15.66:0/1614029193 learned_addr learned my addr 172.21.15.66:0/1614029193 (peer_addr_for_me v2:172.21.15.66:0/0) 2023-11-17T05:32:21.556 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:21.554+0000 7fcb2effd700 1 -- 172.21.15.66:0/1614029193 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fcb20005040 con 0x7fcb300f6510 2023-11-17T05:32:21.556 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:21.554+0000 7fcb2effd700 1 --2- 172.21.15.66:0/1614029193 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fcb300f6510 0x7fcb30108900 secure :-1 s=READY pgs=366 cs=0 l=1 rev1=1 rx=0x7fcb20014000 tx=0x7fcb2000b040).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:32:21.556 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:21.554+0000 7fcb277fe700 1 -- 172.21.15.66:0/1614029193 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7fcb200095f0 con 0x7fcb300f6510 2023-11-17T05:32:21.556 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:21.554+0000 7fcb35a49700 1 -- 172.21.15.66:0/1614029193 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fcb30108ff0 con 0x7fcb300f6510 2023-11-17T05:32:21.556 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:21.555+0000 7fcb277fe700 1 -- 172.21.15.66:0/1614029193 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(25 keys) v1 ==== 1074+0+0 (secure 0 0 0) 0x7fcb20007e70 con 0x7fcb300f6510 2023-11-17T05:32:21.556 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:21.555+0000 7fcb35a49700 1 -- 172.21.15.66:0/1614029193 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fcb30109480 con 0x7fcb300f6510 2023-11-17T05:32:21.556 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:21.555+0000 7fcb277fe700 1 -- 172.21.15.66:0/1614029193 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7fcb20014ea0 con 0x7fcb300f6510 2023-11-17T05:32:21.557 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:21.555+0000 7fcb277fe700 1 -- 172.21.15.66:0/1614029193 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 17) v1 ==== 36269+0+0 (secure 0 0 0) 0x7fcb20025070 con 0x7fcb300f6510 2023-11-17T05:32:21.557 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:21.555+0000 7fcb277fe700 1 --2- 172.21.15.66:0/1614029193 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fcb1c02ffb0 0x7fcb1c032470 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:32:21.557 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:21.555+0000 7fcb277fe700 1 -- 172.21.15.66:0/1614029193 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(5..5 src has 1..5) v4 ==== 1270+0+0 (secure 0 0 0) 0x7fcb20040a70 con 0x7fcb300f6510 2023-11-17T05:32:21.557 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:21.556+0000 7fcb35a49700 1 -- 172.21.15.66:0/1614029193 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fcb30045ad0 con 0x7fcb300f6510 2023-11-17T05:32:21.557 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:21.556+0000 7fcb2e7fc700 1 --2- 172.21.15.66:0/1614029193 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fcb1c02ffb0 0x7fcb1c032470 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:32:21.557 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:21.556+0000 7fcb2e7fc700 1 --2- 172.21.15.66:0/1614029193 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fcb1c02ffb0 0x7fcb1c032470 secure :-1 s=READY pgs=148 cs=0 l=1 rev1=1 rx=0x7fcb180096a0 tx=0x7fcb18006b40).ready entity=mgr.14164 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:32:21.560 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:21.560+0000 7fcb277fe700 1 -- 172.21.15.66:0/1614029193 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+152212 (secure 0 0 0) 0x7fcb20026350 con 0x7fcb300f6510 2023-11-17T05:32:21.784 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:21.784+0000 7fcb35a49700 1 -- 172.21.15.66:0/1614029193 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "mon dump", "format": "json"} v 0) v1 -- 0x7fcb3010a990 con 0x7fcb300f6510 2023-11-17T05:32:21.784 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:21.784+0000 7fcb277fe700 1 -- 172.21.15.66:0/1614029193 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mon_command_ack([{"prefix": "mon dump", "format": "json"}]=0 dumped monmap epoch 1 v1) v1 ==== 95+0+690 (secure 0 0 0) 0x7fcb20012330 con 0x7fcb300f6510 2023-11-17T05:32:21.785 INFO:teuthology.orchestra.run.smithi066.stdout: 2023-11-17T05:32:21.785 INFO:teuthology.orchestra.run.smithi066.stdout:{"epoch":1,"fsid":"ed6470e2-8508-11ee-8dbe-212e2dc638e7","modified":"2023-11-17T05:20:13.934038Z","created":"2023-11-17T05:20:13.934038Z","min_mon_release":16,"min_mon_release_name":"pacific","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging"],"optional":[]},"mons":[{"rank":0,"name":"smithi040","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:3300","nonce":0},{"type":"v1","addr":"172.21.15.40:6789","nonce":0}]},"addr":"172.21.15.40:6789/0","public_addr":"172.21.15.40:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2023-11-17T05:32:21.786 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:21.786+0000 7fcb35a49700 1 -- 172.21.15.66:0/1614029193 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fcb1c02ffb0 msgr2=0x7fcb1c032470 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:32:21.787 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:21.786+0000 7fcb35a49700 1 --2- 172.21.15.66:0/1614029193 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fcb1c02ffb0 0x7fcb1c032470 secure :-1 s=READY pgs=148 cs=0 l=1 rev1=1 rx=0x7fcb180096a0 tx=0x7fcb18006b40).stop 2023-11-17T05:32:21.787 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:21.786+0000 7fcb35a49700 1 -- 172.21.15.66:0/1614029193 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fcb300f6510 msgr2=0x7fcb30108900 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:32:21.787 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:21.786+0000 7fcb35a49700 1 --2- 172.21.15.66:0/1614029193 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fcb300f6510 0x7fcb30108900 secure :-1 s=READY pgs=366 cs=0 l=1 rev1=1 rx=0x7fcb20014000 tx=0x7fcb2000b040).stop 2023-11-17T05:32:21.787 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:21.786+0000 7fcb35a49700 1 -- 172.21.15.66:0/1614029193 shutdown_connections 2023-11-17T05:32:21.787 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:21.786+0000 7fcb35a49700 1 --2- 172.21.15.66:0/1614029193 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fcb1c02ffb0 0x7fcb1c032470 unknown :-1 s=CLOSED pgs=148 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:32:21.787 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:21.786+0000 7fcb35a49700 1 --2- 172.21.15.66:0/1614029193 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fcb300f6510 0x7fcb30108900 unknown :-1 s=CLOSED pgs=366 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:32:21.787 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:21.786+0000 7fcb35a49700 1 -- 172.21.15.66:0/1614029193 >> 172.21.15.66:0/1614029193 conn(0x7fcb300f1c70 msgr2=0x7fcb300f4030 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:32:21.787 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:21.786+0000 7fcb35a49700 1 -- 172.21.15.66:0/1614029193 shutdown_connections 2023-11-17T05:32:21.787 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:21.786+0000 7fcb35a49700 1 -- 172.21.15.66:0/1614029193 wait complete. 2023-11-17T05:32:21.788 INFO:teuthology.orchestra.run.smithi066.stderr:dumped monmap epoch 1 2023-11-17T05:32:21.989 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:32:21 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:32:21.524413+0000 mgr.smithi040.ogcprl (mgr.14164) 346 : cluster [DBG] pgmap v330: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:32:22.989 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:32:22 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:32:21.785166+0000 mon.smithi040 (mon.0) 295 : audit [DBG] from='client.? 172.21.15.66:0/1614029193' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-11-17T05:32:23.189 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2023-11-17T05:32:23.189 DEBUG:teuthology.orchestra.run.smithi066:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.4 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ed6470e2-8508-11ee-8dbe-212e2dc638e7 -- ceph mon dump -f json 2023-11-17T05:32:23.988 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:32:23 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:32:23.524691+0000 mgr.smithi040.ogcprl (mgr.14164) 347 : cluster [DBG] pgmap v331: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:32:24.396 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:24.395+0000 7f0e02cf6700 1 -- 172.21.15.66:0/3441042876 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f0dfc060280 msgr2=0x7f0dfc0f90f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:32:24.396 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:24.395+0000 7f0e02cf6700 1 --2- 172.21.15.66:0/3441042876 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f0dfc060280 0x7f0dfc0f90f0 secure :-1 s=READY pgs=367 cs=0 l=1 rev1=1 rx=0x7f0df0009fb0 tx=0x7f0df0009b30).stop 2023-11-17T05:32:24.396 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:24.395+0000 7f0e02cf6700 1 -- 172.21.15.66:0/3441042876 shutdown_connections 2023-11-17T05:32:24.396 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:24.395+0000 7f0e02cf6700 1 --2- 172.21.15.66:0/3441042876 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f0dfc060280 0x7f0dfc0f90f0 unknown :-1 s=CLOSED pgs=367 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:32:24.396 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:24.395+0000 7f0e02cf6700 1 -- 172.21.15.66:0/3441042876 >> 172.21.15.66:0/3441042876 conn(0x7f0dfc0f0b70 msgr2=0x7f0dfc0f2f90 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:32:24.396 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:24.395+0000 7f0e02cf6700 1 -- 172.21.15.66:0/3441042876 shutdown_connections 2023-11-17T05:32:24.396 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:24.395+0000 7f0e02cf6700 1 -- 172.21.15.66:0/3441042876 wait complete. 2023-11-17T05:32:24.396 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:24.395+0000 7f0e02cf6700 1 Processor -- start 2023-11-17T05:32:24.396 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:24.395+0000 7f0e02cf6700 1 -- start start 2023-11-17T05:32:24.396 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:24.396+0000 7f0e02cf6700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f0dfc060280 0x7f0dfc0f6b70 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:32:24.396 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:24.396+0000 7f0e02cf6700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f0dfc0f4ff0 con 0x7f0dfc060280 2023-11-17T05:32:24.398 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:24.396+0000 7f0e00a92700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f0dfc060280 0x7f0dfc0f6b70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:32:24.398 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:24.396+0000 7f0e00a92700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f0dfc060280 0x7f0dfc0f6b70 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.66:51288/0 (socket says 172.21.15.66:51288) 2023-11-17T05:32:24.398 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:24.396+0000 7f0e00a92700 1 -- 172.21.15.66:0/1444878413 learned_addr learned my addr 172.21.15.66:0/1444878413 (peer_addr_for_me v2:172.21.15.66:0/0) 2023-11-17T05:32:24.398 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:24.396+0000 7f0e00a92700 1 -- 172.21.15.66:0/1444878413 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f0df0005040 con 0x7f0dfc060280 2023-11-17T05:32:24.399 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:24.396+0000 7f0e00a92700 1 --2- 172.21.15.66:0/1444878413 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f0dfc060280 0x7f0dfc0f6b70 secure :-1 s=READY pgs=368 cs=0 l=1 rev1=1 rx=0x7f0df0013510 tx=0x7f0df000b040).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:32:24.399 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:24.396+0000 7f0df97fa700 1 -- 172.21.15.66:0/1444878413 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f0df0009320 con 0x7f0dfc060280 2023-11-17T05:32:24.399 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:24.396+0000 7f0e02cf6700 1 -- 172.21.15.66:0/1444878413 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f0dfc0f5190 con 0x7f0dfc060280 2023-11-17T05:32:24.399 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:24.397+0000 7f0df97fa700 1 -- 172.21.15.66:0/1444878413 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(25 keys) v1 ==== 1074+0+0 (secure 0 0 0) 0x7f0df0013f00 con 0x7f0dfc060280 2023-11-17T05:32:24.399 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:24.397+0000 7f0df97fa700 1 -- 172.21.15.66:0/1444878413 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f0df0014dc0 con 0x7f0dfc060280 2023-11-17T05:32:24.399 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:24.397+0000 7f0e02cf6700 1 -- 172.21.15.66:0/1444878413 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f0dfc0f5620 con 0x7f0dfc060280 2023-11-17T05:32:24.399 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:24.397+0000 7f0df97fa700 1 -- 172.21.15.66:0/1444878413 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 17) v1 ==== 36269+0+0 (secure 0 0 0) 0x7f0df00095f0 con 0x7f0dfc060280 2023-11-17T05:32:24.399 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:24.397+0000 7f0e02cf6700 1 -- 172.21.15.66:0/1444878413 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f0dfc059100 con 0x7f0dfc060280 2023-11-17T05:32:24.399 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:24.398+0000 7f0df97fa700 1 --2- 172.21.15.66:0/1444878413 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f0dec02ff60 0x7f0dec032420 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:32:24.399 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:24.398+0000 7f0df97fa700 1 -- 172.21.15.66:0/1444878413 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(5..5 src has 1..5) v4 ==== 1270+0+0 (secure 0 0 0) 0x7f0df0041000 con 0x7f0dfc060280 2023-11-17T05:32:24.399 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:24.398+0000 7f0dfbfff700 1 --2- 172.21.15.66:0/1444878413 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f0dec02ff60 0x7f0dec032420 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:32:24.399 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:24.398+0000 7f0dfbfff700 1 --2- 172.21.15.66:0/1444878413 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f0dec02ff60 0x7f0dec032420 secure :-1 s=READY pgs=149 cs=0 l=1 rev1=1 rx=0x7f0de80096a0 tx=0x7f0de8006b40).ready entity=mgr.14164 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:32:24.402 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:24.402+0000 7f0df97fa700 1 -- 172.21.15.66:0/1444878413 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+152212 (secure 0 0 0) 0x7f0df0014420 con 0x7f0dfc060280 2023-11-17T05:32:24.622 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:24.621+0000 7f0e02cf6700 1 -- 172.21.15.66:0/1444878413 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "mon dump", "format": "json"} v 0) v1 -- 0x7f0dfc0fbf50 con 0x7f0dfc060280 2023-11-17T05:32:24.622 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:24.622+0000 7f0df97fa700 1 -- 172.21.15.66:0/1444878413 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mon_command_ack([{"prefix": "mon dump", "format": "json"}]=0 dumped monmap epoch 1 v1) v1 ==== 95+0+690 (secure 0 0 0) 0x7f0df0010230 con 0x7f0dfc060280 2023-11-17T05:32:24.622 INFO:teuthology.orchestra.run.smithi066.stdout: 2023-11-17T05:32:24.622 INFO:teuthology.orchestra.run.smithi066.stdout:{"epoch":1,"fsid":"ed6470e2-8508-11ee-8dbe-212e2dc638e7","modified":"2023-11-17T05:20:13.934038Z","created":"2023-11-17T05:20:13.934038Z","min_mon_release":16,"min_mon_release_name":"pacific","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging"],"optional":[]},"mons":[{"rank":0,"name":"smithi040","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:3300","nonce":0},{"type":"v1","addr":"172.21.15.40:6789","nonce":0}]},"addr":"172.21.15.40:6789/0","public_addr":"172.21.15.40:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2023-11-17T05:32:24.623 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:24.623+0000 7f0e02cf6700 1 -- 172.21.15.66:0/1444878413 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f0dec02ff60 msgr2=0x7f0dec032420 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:32:24.624 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:24.623+0000 7f0e02cf6700 1 --2- 172.21.15.66:0/1444878413 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f0dec02ff60 0x7f0dec032420 secure :-1 s=READY pgs=149 cs=0 l=1 rev1=1 rx=0x7f0de80096a0 tx=0x7f0de8006b40).stop 2023-11-17T05:32:24.624 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:24.623+0000 7f0e02cf6700 1 -- 172.21.15.66:0/1444878413 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f0dfc060280 msgr2=0x7f0dfc0f6b70 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:32:24.624 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:24.623+0000 7f0e02cf6700 1 --2- 172.21.15.66:0/1444878413 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f0dfc060280 0x7f0dfc0f6b70 secure :-1 s=READY pgs=368 cs=0 l=1 rev1=1 rx=0x7f0df0013510 tx=0x7f0df000b040).stop 2023-11-17T05:32:24.624 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:24.623+0000 7f0e02cf6700 1 -- 172.21.15.66:0/1444878413 shutdown_connections 2023-11-17T05:32:24.624 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:24.623+0000 7f0e02cf6700 1 --2- 172.21.15.66:0/1444878413 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f0dec02ff60 0x7f0dec032420 unknown :-1 s=CLOSED pgs=149 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:32:24.624 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:24.623+0000 7f0e02cf6700 1 --2- 172.21.15.66:0/1444878413 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f0dfc060280 0x7f0dfc0f6b70 unknown :-1 s=CLOSED pgs=368 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:32:24.624 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:24.623+0000 7f0e02cf6700 1 -- 172.21.15.66:0/1444878413 >> 172.21.15.66:0/1444878413 conn(0x7f0dfc0f0b70 msgr2=0x7f0dfc0f1660 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:32:24.624 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:24.623+0000 7f0e02cf6700 1 -- 172.21.15.66:0/1444878413 shutdown_connections 2023-11-17T05:32:24.624 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:24.623+0000 7f0e02cf6700 1 -- 172.21.15.66:0/1444878413 wait complete. 2023-11-17T05:32:24.624 INFO:teuthology.orchestra.run.smithi066.stderr:dumped monmap epoch 1 2023-11-17T05:32:25.988 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:32:25 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:32:24.622284+0000 mon.smithi040 (mon.0) 296 : audit [DBG] from='client.? 172.21.15.66:0/1444878413' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-11-17T05:32:26.016 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2023-11-17T05:32:26.016 DEBUG:teuthology.orchestra.run.smithi066:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.4 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ed6470e2-8508-11ee-8dbe-212e2dc638e7 -- ceph mon dump -f json 2023-11-17T05:32:26.989 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:32:26 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:32:25.525064+0000 mgr.smithi040.ogcprl (mgr.14164) 348 : cluster [DBG] pgmap v332: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:32:27.229 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:27.222+0000 7fee01c18700 1 -- 172.21.15.66:0/2367529165 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fedfc060200 msgr2=0x7fedfc0fb570 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:32:27.229 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:27.222+0000 7fee01c18700 1 --2- 172.21.15.66:0/2367529165 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fedfc060200 0x7fedfc0fb570 secure :-1 s=READY pgs=369 cs=0 l=1 rev1=1 rx=0x7fedec009fb0 tx=0x7fedec009b30).stop 2023-11-17T05:32:27.229 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:27.222+0000 7fee01c18700 1 -- 172.21.15.66:0/2367529165 shutdown_connections 2023-11-17T05:32:27.229 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:27.222+0000 7fee01c18700 1 --2- 172.21.15.66:0/2367529165 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fedfc060200 0x7fedfc0fb570 unknown :-1 s=CLOSED pgs=369 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:32:27.229 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:27.222+0000 7fee01c18700 1 -- 172.21.15.66:0/2367529165 >> 172.21.15.66:0/2367529165 conn(0x7fedfc0f0b70 msgr2=0x7fedfc0f2f90 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:32:27.229 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:27.222+0000 7fee01c18700 1 -- 172.21.15.66:0/2367529165 shutdown_connections 2023-11-17T05:32:27.229 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:27.222+0000 7fee01c18700 1 -- 172.21.15.66:0/2367529165 wait complete. 2023-11-17T05:32:27.229 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:27.223+0000 7fee01c18700 1 Processor -- start 2023-11-17T05:32:27.230 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:27.223+0000 7fee01c18700 1 -- start start 2023-11-17T05:32:27.230 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:27.223+0000 7fee01c18700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fedfc060200 0x7fedfc1088d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:32:27.230 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:27.223+0000 7fee01c18700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fedfc108dc0 con 0x7fedfc060200 2023-11-17T05:32:27.230 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:27.223+0000 7fedfb7fe700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fedfc060200 0x7fedfc1088d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:32:27.230 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:27.223+0000 7fedfb7fe700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fedfc060200 0x7fedfc1088d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.66:51312/0 (socket says 172.21.15.66:51312) 2023-11-17T05:32:27.230 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:27.223+0000 7fedfb7fe700 1 -- 172.21.15.66:0/460215288 learned_addr learned my addr 172.21.15.66:0/460215288 (peer_addr_for_me v2:172.21.15.66:0/0) 2023-11-17T05:32:27.230 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:27.223+0000 7fedfb7fe700 1 -- 172.21.15.66:0/460215288 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fedec005040 con 0x7fedfc060200 2023-11-17T05:32:27.230 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:27.224+0000 7fedfb7fe700 1 --2- 172.21.15.66:0/460215288 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fedfc060200 0x7fedfc1088d0 secure :-1 s=READY pgs=370 cs=0 l=1 rev1=1 rx=0x7fedec013510 tx=0x7fedec00b040).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:32:27.230 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:27.224+0000 7fedf3fff700 1 -- 172.21.15.66:0/460215288 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7fedec009320 con 0x7fedfc060200 2023-11-17T05:32:27.231 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:27.224+0000 7fee01c18700 1 -- 172.21.15.66:0/460215288 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fedfc108fc0 con 0x7fedfc060200 2023-11-17T05:32:27.231 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:27.224+0000 7fedf3fff700 1 -- 172.21.15.66:0/460215288 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(25 keys) v1 ==== 1074+0+0 (secure 0 0 0) 0x7fedec013f00 con 0x7fedfc060200 2023-11-17T05:32:27.231 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:27.224+0000 7fedf3fff700 1 -- 172.21.15.66:0/460215288 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7fedec014dc0 con 0x7fedfc060200 2023-11-17T05:32:27.231 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:27.224+0000 7fee01c18700 1 -- 172.21.15.66:0/460215288 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fedfc109450 con 0x7fedfc060200 2023-11-17T05:32:27.231 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:27.225+0000 7fedf3fff700 1 -- 172.21.15.66:0/460215288 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 17) v1 ==== 36269+0+0 (secure 0 0 0) 0x7fedec0095f0 con 0x7fedfc060200 2023-11-17T05:32:27.231 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:27.225+0000 7fee01c18700 1 -- 172.21.15.66:0/460215288 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fedfc045ad0 con 0x7fedfc060200 2023-11-17T05:32:27.231 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:27.225+0000 7fedf3fff700 1 --2- 172.21.15.66:0/460215288 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fede802ffb0 0x7fede8032470 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:32:27.231 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:27.225+0000 7fedf3fff700 1 -- 172.21.15.66:0/460215288 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(5..5 src has 1..5) v4 ==== 1270+0+0 (secure 0 0 0) 0x7fedec040f90 con 0x7fedfc060200 2023-11-17T05:32:27.231 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:27.225+0000 7fedfaffd700 1 --2- 172.21.15.66:0/460215288 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fede802ffb0 0x7fede8032470 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:32:27.231 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:27.225+0000 7fedfaffd700 1 --2- 172.21.15.66:0/460215288 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fede802ffb0 0x7fede8032470 secure :-1 s=READY pgs=150 cs=0 l=1 rev1=1 rx=0x7fede40096a0 tx=0x7fede4006b40).ready entity=mgr.14164 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:32:27.232 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:27.230+0000 7fedf3fff700 1 -- 172.21.15.66:0/460215288 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+152212 (secure 0 0 0) 0x7fedec01d9d0 con 0x7fedfc060200 2023-11-17T05:32:27.447 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:27.447+0000 7fee01c18700 1 -- 172.21.15.66:0/460215288 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "mon dump", "format": "json"} v 0) v1 -- 0x7fedfc10a300 con 0x7fedfc060200 2023-11-17T05:32:27.448 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:27.448+0000 7fedf3fff700 1 -- 172.21.15.66:0/460215288 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mon_command_ack([{"prefix": "mon dump", "format": "json"}]=0 dumped monmap epoch 1 v1) v1 ==== 95+0+690 (secure 0 0 0) 0x7fedec010230 con 0x7fedfc060200 2023-11-17T05:32:27.448 INFO:teuthology.orchestra.run.smithi066.stdout: 2023-11-17T05:32:27.448 INFO:teuthology.orchestra.run.smithi066.stdout:{"epoch":1,"fsid":"ed6470e2-8508-11ee-8dbe-212e2dc638e7","modified":"2023-11-17T05:20:13.934038Z","created":"2023-11-17T05:20:13.934038Z","min_mon_release":16,"min_mon_release_name":"pacific","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging"],"optional":[]},"mons":[{"rank":0,"name":"smithi040","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:3300","nonce":0},{"type":"v1","addr":"172.21.15.40:6789","nonce":0}]},"addr":"172.21.15.40:6789/0","public_addr":"172.21.15.40:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2023-11-17T05:32:27.450 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:27.450+0000 7fee01c18700 1 -- 172.21.15.66:0/460215288 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fede802ffb0 msgr2=0x7fede8032470 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:32:27.450 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:27.450+0000 7fee01c18700 1 --2- 172.21.15.66:0/460215288 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fede802ffb0 0x7fede8032470 secure :-1 s=READY pgs=150 cs=0 l=1 rev1=1 rx=0x7fede40096a0 tx=0x7fede4006b40).stop 2023-11-17T05:32:27.450 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:27.450+0000 7fee01c18700 1 -- 172.21.15.66:0/460215288 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fedfc060200 msgr2=0x7fedfc1088d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:32:27.450 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:27.450+0000 7fee01c18700 1 --2- 172.21.15.66:0/460215288 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fedfc060200 0x7fedfc1088d0 secure :-1 s=READY pgs=370 cs=0 l=1 rev1=1 rx=0x7fedec013510 tx=0x7fedec00b040).stop 2023-11-17T05:32:27.450 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:27.450+0000 7fee01c18700 1 -- 172.21.15.66:0/460215288 shutdown_connections 2023-11-17T05:32:27.450 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:27.450+0000 7fee01c18700 1 --2- 172.21.15.66:0/460215288 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fede802ffb0 0x7fede8032470 unknown :-1 s=CLOSED pgs=150 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:32:27.450 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:27.450+0000 7fee01c18700 1 --2- 172.21.15.66:0/460215288 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fedfc060200 0x7fedfc1088d0 unknown :-1 s=CLOSED pgs=370 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:32:27.450 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:27.450+0000 7fee01c18700 1 -- 172.21.15.66:0/460215288 >> 172.21.15.66:0/460215288 conn(0x7fedfc0f0b70 msgr2=0x7fedfc0f1670 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:32:27.450 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:27.450+0000 7fee01c18700 1 -- 172.21.15.66:0/460215288 shutdown_connections 2023-11-17T05:32:27.451 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:27.450+0000 7fee01c18700 1 -- 172.21.15.66:0/460215288 wait complete. 2023-11-17T05:32:27.451 INFO:teuthology.orchestra.run.smithi066.stderr:dumped monmap epoch 1 2023-11-17T05:32:27.990 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:32:27 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:32:27.448455+0000 mon.smithi040 (mon.0) 297 : audit [DBG] from='client.? 172.21.15.66:0/460215288' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-11-17T05:32:28.848 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2023-11-17T05:32:28.848 DEBUG:teuthology.orchestra.run.smithi066:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.4 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ed6470e2-8508-11ee-8dbe-212e2dc638e7 -- ceph mon dump -f json 2023-11-17T05:32:28.989 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:32:28 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:32:27.525373+0000 mgr.smithi040.ogcprl (mgr.14164) 349 : cluster [DBG] pgmap v333: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:32:30.000 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:29.997+0000 7f4d6222e700 1 -- 172.21.15.66:0/726401984 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f4d5c0f51a0 msgr2=0x7f4d5c0f55c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:32:30.000 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:29.997+0000 7f4d6222e700 1 --2- 172.21.15.66:0/726401984 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f4d5c0f51a0 0x7f4d5c0f55c0 secure :-1 s=READY pgs=371 cs=0 l=1 rev1=1 rx=0x7f4d58009fb0 tx=0x7f4d58009b30).stop 2023-11-17T05:32:30.000 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:29.997+0000 7f4d6222e700 1 -- 172.21.15.66:0/726401984 shutdown_connections 2023-11-17T05:32:30.000 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:29.997+0000 7f4d6222e700 1 --2- 172.21.15.66:0/726401984 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f4d5c0f51a0 0x7f4d5c0f55c0 unknown :-1 s=CLOSED pgs=371 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:32:30.000 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:29.997+0000 7f4d6222e700 1 -- 172.21.15.66:0/726401984 >> 172.21.15.66:0/726401984 conn(0x7f4d5c0f09a0 msgr2=0x7f4d5c0f2dc0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:32:30.000 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:29.998+0000 7f4d6222e700 1 -- 172.21.15.66:0/726401984 shutdown_connections 2023-11-17T05:32:30.000 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:29.998+0000 7f4d6222e700 1 -- 172.21.15.66:0/726401984 wait complete. 2023-11-17T05:32:30.000 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:29.998+0000 7f4d6222e700 1 Processor -- start 2023-11-17T05:32:30.000 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:29.998+0000 7f4d6222e700 1 -- start start 2023-11-17T05:32:30.000 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:29.998+0000 7f4d6222e700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f4d5c0f51a0 0x7f4d5c1091d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:32:30.001 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:29.998+0000 7f4d6222e700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f4d5c060220 con 0x7f4d5c0f51a0 2023-11-17T05:32:30.001 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:29.999+0000 7f4d6122c700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f4d5c0f51a0 0x7f4d5c1091d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:32:30.001 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:29.999+0000 7f4d6122c700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f4d5c0f51a0 0x7f4d5c1091d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.66:51330/0 (socket says 172.21.15.66:51330) 2023-11-17T05:32:30.001 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:29.999+0000 7f4d6122c700 1 -- 172.21.15.66:0/3023283425 learned_addr learned my addr 172.21.15.66:0/3023283425 (peer_addr_for_me v2:172.21.15.66:0/0) 2023-11-17T05:32:30.001 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:29.999+0000 7f4d6122c700 1 -- 172.21.15.66:0/3023283425 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f4d58005040 con 0x7f4d5c0f51a0 2023-11-17T05:32:30.001 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:29.999+0000 7f4d6122c700 1 --2- 172.21.15.66:0/3023283425 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f4d5c0f51a0 0x7f4d5c1091d0 secure :-1 s=READY pgs=372 cs=0 l=1 rev1=1 rx=0x7f4d58013510 tx=0x7f4d5800b040).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:32:30.001 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:29.999+0000 7f4d4dffb700 1 -- 172.21.15.66:0/3023283425 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f4d58009320 con 0x7f4d5c0f51a0 2023-11-17T05:32:30.001 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:29.999+0000 7f4d4dffb700 1 -- 172.21.15.66:0/3023283425 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(25 keys) v1 ==== 1074+0+0 (secure 0 0 0) 0x7f4d58013f00 con 0x7f4d5c0f51a0 2023-11-17T05:32:30.001 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:29.999+0000 7f4d6222e700 1 -- 172.21.15.66:0/3023283425 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f4d5c060420 con 0x7f4d5c0f51a0 2023-11-17T05:32:30.001 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:29.999+0000 7f4d4dffb700 1 -- 172.21.15.66:0/3023283425 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f4d58014dc0 con 0x7f4d5c0f51a0 2023-11-17T05:32:30.001 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:29.999+0000 7f4d6222e700 1 -- 172.21.15.66:0/3023283425 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f4d5c109880 con 0x7f4d5c0f51a0 2023-11-17T05:32:30.002 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:30.000+0000 7f4d4dffb700 1 -- 172.21.15.66:0/3023283425 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 17) v1 ==== 36269+0+0 (secure 0 0 0) 0x7f4d580095f0 con 0x7f4d5c0f51a0 2023-11-17T05:32:30.002 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:30.000+0000 7f4d6222e700 1 -- 172.21.15.66:0/3023283425 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f4d5c059100 con 0x7f4d5c0f51a0 2023-11-17T05:32:30.002 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:30.000+0000 7f4d4dffb700 1 --2- 172.21.15.66:0/3023283425 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f4d4802ff60 0x7f4d48032420 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:32:30.002 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:30.001+0000 7f4d4dffb700 1 -- 172.21.15.66:0/3023283425 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(5..5 src has 1..5) v4 ==== 1270+0+0 (secure 0 0 0) 0x7f4d58041020 con 0x7f4d5c0f51a0 2023-11-17T05:32:30.002 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:30.001+0000 7f4d60a2b700 1 --2- 172.21.15.66:0/3023283425 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f4d4802ff60 0x7f4d48032420 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:32:30.002 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:30.001+0000 7f4d60a2b700 1 --2- 172.21.15.66:0/3023283425 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f4d4802ff60 0x7f4d48032420 secure :-1 s=READY pgs=151 cs=0 l=1 rev1=1 rx=0x7f4d500096a0 tx=0x7f4d50006b40).ready entity=mgr.14164 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:32:30.005 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:30.005+0000 7f4d4dffb700 1 -- 172.21.15.66:0/3023283425 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+152212 (secure 0 0 0) 0x7f4d58014420 con 0x7f4d5c0f51a0 2023-11-17T05:32:30.227 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:30.227+0000 7f4d6222e700 1 -- 172.21.15.66:0/3023283425 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "mon dump", "format": "json"} v 0) v1 -- 0x7f4d5c0fabe0 con 0x7f4d5c0f51a0 2023-11-17T05:32:30.228 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:30.228+0000 7f4d4dffb700 1 -- 172.21.15.66:0/3023283425 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mon_command_ack([{"prefix": "mon dump", "format": "json"}]=0 dumped monmap epoch 1 v1) v1 ==== 95+0+690 (secure 0 0 0) 0x7f4d5801c920 con 0x7f4d5c0f51a0 2023-11-17T05:32:30.228 INFO:teuthology.orchestra.run.smithi066.stdout: 2023-11-17T05:32:30.228 INFO:teuthology.orchestra.run.smithi066.stdout:{"epoch":1,"fsid":"ed6470e2-8508-11ee-8dbe-212e2dc638e7","modified":"2023-11-17T05:20:13.934038Z","created":"2023-11-17T05:20:13.934038Z","min_mon_release":16,"min_mon_release_name":"pacific","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging"],"optional":[]},"mons":[{"rank":0,"name":"smithi040","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:3300","nonce":0},{"type":"v1","addr":"172.21.15.40:6789","nonce":0}]},"addr":"172.21.15.40:6789/0","public_addr":"172.21.15.40:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2023-11-17T05:32:30.230 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:30.229+0000 7f4d6222e700 1 -- 172.21.15.66:0/3023283425 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f4d4802ff60 msgr2=0x7f4d48032420 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:32:30.230 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:30.229+0000 7f4d6222e700 1 --2- 172.21.15.66:0/3023283425 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f4d4802ff60 0x7f4d48032420 secure :-1 s=READY pgs=151 cs=0 l=1 rev1=1 rx=0x7f4d500096a0 tx=0x7f4d50006b40).stop 2023-11-17T05:32:30.230 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:30.229+0000 7f4d6222e700 1 -- 172.21.15.66:0/3023283425 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f4d5c0f51a0 msgr2=0x7f4d5c1091d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:32:30.230 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:30.229+0000 7f4d6222e700 1 --2- 172.21.15.66:0/3023283425 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f4d5c0f51a0 0x7f4d5c1091d0 secure :-1 s=READY pgs=372 cs=0 l=1 rev1=1 rx=0x7f4d58013510 tx=0x7f4d5800b040).stop 2023-11-17T05:32:30.230 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:30.230+0000 7f4d6222e700 1 -- 172.21.15.66:0/3023283425 shutdown_connections 2023-11-17T05:32:30.230 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:30.230+0000 7f4d6222e700 1 --2- 172.21.15.66:0/3023283425 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f4d4802ff60 0x7f4d48032420 unknown :-1 s=CLOSED pgs=151 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:32:30.230 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:30.230+0000 7f4d6222e700 1 --2- 172.21.15.66:0/3023283425 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f4d5c0f51a0 0x7f4d5c1091d0 unknown :-1 s=CLOSED pgs=372 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:32:30.230 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:30.230+0000 7f4d6222e700 1 -- 172.21.15.66:0/3023283425 >> 172.21.15.66:0/3023283425 conn(0x7f4d5c0f09a0 msgr2=0x7f4d5c0f1460 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:32:30.230 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:30.230+0000 7f4d6222e700 1 -- 172.21.15.66:0/3023283425 shutdown_connections 2023-11-17T05:32:30.231 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:32:30.230+0000 7f4d6222e700 1 -- 172.21.15.66:0/3023283425 wait complete. 2023-11-17T05:32:30.231 INFO:teuthology.orchestra.run.smithi066.stderr:dumped monmap epoch 1 2023-11-17T05:32:30.988 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:32:30 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:32:29.525660+0000 mgr.smithi040.ogcprl (mgr.14164) 350 : cluster [DBG] pgmap v334: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:33:26.699 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:32:30 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:32:30.228644+0000 mon.smithi040 (mon.0) 298 : audit [DBG] from='client.? 172.21.15.66:0/3023283425' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-11-17T05:33:26.700 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2023-11-17T05:33:26.701 DEBUG:teuthology.orchestra.run.smithi066:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.4 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ed6470e2-8508-11ee-8dbe-212e2dc638e7 -- ceph mon dump -f json 2023-11-17T05:33:26.730 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:32:31 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:32:31.526050+0000 mgr.smithi040.ogcprl (mgr.14164) 351 : cluster [DBG] pgmap v335: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:33:26.741 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:32:33 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:32:33.526317+0000 mgr.smithi040.ogcprl (mgr.14164) 352 : cluster [DBG] pgmap v336: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:33:26.741 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:32:35 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:32:35.526681+0000 mgr.smithi040.ogcprl (mgr.14164) 353 : cluster [DBG] pgmap v337: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:33:26.784 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:32:37 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:32:37.527030+0000 mgr.smithi040.ogcprl (mgr.14164) 354 : cluster [DBG] pgmap v338: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:33:26.784 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:32:40 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:32:39.527270+0000 mgr.smithi040.ogcprl (mgr.14164) 355 : cluster [DBG] pgmap v339: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:33:26.784 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:32:41 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:32:41.527567+0000 mgr.smithi040.ogcprl (mgr.14164) 356 : cluster [DBG] pgmap v340: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:33:26.784 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:32:43 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:32:43.527901+0000 mgr.smithi040.ogcprl (mgr.14164) 357 : cluster [DBG] pgmap v341: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:33:26.785 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:32:45 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:32:45.528190+0000 mgr.smithi040.ogcprl (mgr.14164) 358 : cluster [DBG] pgmap v342: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:33:26.785 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:32:47 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:32:47.528546+0000 mgr.smithi040.ogcprl (mgr.14164) 359 : cluster [DBG] pgmap v343: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:33:26.785 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:32:50 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:32:49.528929+0000 mgr.smithi040.ogcprl (mgr.14164) 360 : cluster [DBG] pgmap v344: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:33:26.785 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:32:51 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:32:51.529300+0000 mgr.smithi040.ogcprl (mgr.14164) 361 : cluster [DBG] pgmap v345: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:33:26.785 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:32:53 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:32:53.529584+0000 mgr.smithi040.ogcprl (mgr.14164) 362 : cluster [DBG] pgmap v346: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:33:26.785 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:32:55 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:32:55.529850+0000 mgr.smithi040.ogcprl (mgr.14164) 363 : cluster [DBG] pgmap v347: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:33:26.786 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:32:57 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:32:57.530207+0000 mgr.smithi040.ogcprl (mgr.14164) 364 : cluster [DBG] pgmap v348: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:33:26.786 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:33:00 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:32:59.530598+0000 mgr.smithi040.ogcprl (mgr.14164) 365 : cluster [DBG] pgmap v349: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:33:26.786 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:33:01 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:33:01.531010+0000 mgr.smithi040.ogcprl (mgr.14164) 366 : cluster [DBG] pgmap v350: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:33:26.786 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:33:03 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:33:03.531340+0000 mgr.smithi040.ogcprl (mgr.14164) 367 : cluster [DBG] pgmap v351: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:33:26.786 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:33:05 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:33:05.531677+0000 mgr.smithi040.ogcprl (mgr.14164) 368 : cluster [DBG] pgmap v352: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:33:26.787 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:33:07 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:33:07.532042+0000 mgr.smithi040.ogcprl (mgr.14164) 369 : cluster [DBG] pgmap v353: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:33:26.787 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:33:10 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:33:09.532450+0000 mgr.smithi040.ogcprl (mgr.14164) 370 : cluster [DBG] pgmap v354: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:33:26.787 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:33:11 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:33:11.321073+0000 mon.smithi040 (mon.0) 299 : audit [INF] from='mgr.14164 172.21.15.40:0/2203032451' entity='mgr.smithi040.ogcprl' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi040.ogcprl/mirror_snapshot_schedule"}]: dispatch 2023-11-17T05:33:26.787 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:33:11 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:33:11.321291+0000 mon.smithi040 (mon.0) 300 : audit [INF] from='mgr.14164 172.21.15.40:0/2203032451' entity='mgr.smithi040.ogcprl' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi040.ogcprl/trash_purge_schedule"}]: dispatch 2023-11-17T05:33:26.787 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:33:12 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:33:11.532787+0000 mgr.smithi040.ogcprl (mgr.14164) 371 : cluster [DBG] pgmap v355: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:33:26.787 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:33:13 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:33:13.533092+0000 mgr.smithi040.ogcprl (mgr.14164) 372 : cluster [DBG] pgmap v356: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:33:26.788 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:33:15 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:33:15.533352+0000 mgr.smithi040.ogcprl (mgr.14164) 373 : cluster [DBG] pgmap v357: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:33:26.788 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:33:17 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:33:17.533647+0000 mgr.smithi040.ogcprl (mgr.14164) 374 : cluster [DBG] pgmap v358: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:33:26.788 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:33:20 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:33:19.533988+0000 mgr.smithi040.ogcprl (mgr.14164) 375 : cluster [DBG] pgmap v359: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:33:26.788 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:33:21 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:33:21.534307+0000 mgr.smithi040.ogcprl (mgr.14164) 376 : cluster [DBG] pgmap v360: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:33:26.788 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:33:23 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:33:23.534670+0000 mgr.smithi040.ogcprl (mgr.14164) 377 : cluster [DBG] pgmap v361: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:33:26.788 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:33:25 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:33:25.535057+0000 mgr.smithi040.ogcprl (mgr.14164) 378 : cluster [DBG] pgmap v362: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:33:27.951 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:27.948+0000 7f9cd7bd4700 1 -- 172.21.15.66:0/3284637773 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f9cd0060280 msgr2=0x7f9cd00f90f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:33:27.951 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:27.948+0000 7f9cd7bd4700 1 --2- 172.21.15.66:0/3284637773 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f9cd0060280 0x7f9cd00f90f0 secure :-1 s=READY pgs=373 cs=0 l=1 rev1=1 rx=0x7f9ccc009fb0 tx=0x7f9ccc009b30).stop 2023-11-17T05:33:27.952 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:27.949+0000 7f9cd7bd4700 1 -- 172.21.15.66:0/3284637773 shutdown_connections 2023-11-17T05:33:27.952 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:27.949+0000 7f9cd7bd4700 1 --2- 172.21.15.66:0/3284637773 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f9cd0060280 0x7f9cd00f90f0 unknown :-1 s=CLOSED pgs=373 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:33:27.952 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:27.949+0000 7f9cd7bd4700 1 -- 172.21.15.66:0/3284637773 >> 172.21.15.66:0/3284637773 conn(0x7f9cd00f0b70 msgr2=0x7f9cd00f2f90 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:33:27.952 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:27.949+0000 7f9cd7bd4700 1 -- 172.21.15.66:0/3284637773 shutdown_connections 2023-11-17T05:33:27.952 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:27.949+0000 7f9cd7bd4700 1 -- 172.21.15.66:0/3284637773 wait complete. 2023-11-17T05:33:27.952 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:27.949+0000 7f9cd7bd4700 1 Processor -- start 2023-11-17T05:33:27.952 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:27.949+0000 7f9cd7bd4700 1 -- start start 2023-11-17T05:33:27.952 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:27.950+0000 7f9cd7bd4700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f9cd0060280 0x7f9cd01066e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:33:27.952 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:27.950+0000 7f9cd7bd4700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f9cd0106bd0 con 0x7f9cd0060280 2023-11-17T05:33:27.952 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:27.950+0000 7f9cd5970700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f9cd0060280 0x7f9cd01066e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:33:27.952 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:27.950+0000 7f9cd5970700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f9cd0060280 0x7f9cd01066e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.66:60312/0 (socket says 172.21.15.66:60312) 2023-11-17T05:33:27.952 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:27.950+0000 7f9cd5970700 1 -- 172.21.15.66:0/3065665150 learned_addr learned my addr 172.21.15.66:0/3065665150 (peer_addr_for_me v2:172.21.15.66:0/0) 2023-11-17T05:33:27.953 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:27.950+0000 7f9cd5970700 1 -- 172.21.15.66:0/3065665150 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f9ccc005040 con 0x7f9cd0060280 2023-11-17T05:33:27.953 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:27.950+0000 7f9cd5970700 1 --2- 172.21.15.66:0/3065665150 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f9cd0060280 0x7f9cd01066e0 secure :-1 s=READY pgs=374 cs=0 l=1 rev1=1 rx=0x7f9ccc013510 tx=0x7f9ccc00b040).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:33:27.954 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:27.950+0000 7f9cc27fc700 1 -- 172.21.15.66:0/3065665150 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f9ccc009320 con 0x7f9cd0060280 2023-11-17T05:33:27.954 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:27.950+0000 7f9cc27fc700 1 -- 172.21.15.66:0/3065665150 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(25 keys) v1 ==== 1074+0+0 (secure 0 0 0) 0x7f9ccc013f00 con 0x7f9cd0060280 2023-11-17T05:33:27.954 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:27.950+0000 7f9cd7bd4700 1 -- 172.21.15.66:0/3065665150 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f9cd0106dd0 con 0x7f9cd0060280 2023-11-17T05:33:27.954 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:27.950+0000 7f9cc27fc700 1 -- 172.21.15.66:0/3065665150 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f9ccc014dc0 con 0x7f9cd0060280 2023-11-17T05:33:27.954 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:27.951+0000 7f9cd7bd4700 1 -- 172.21.15.66:0/3065665150 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f9cd0107260 con 0x7f9cd0060280 2023-11-17T05:33:27.954 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:27.951+0000 7f9cc27fc700 1 -- 172.21.15.66:0/3065665150 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 17) v1 ==== 36269+0+0 (secure 0 0 0) 0x7f9ccc0095f0 con 0x7f9cd0060280 2023-11-17T05:33:27.954 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:27.952+0000 7f9cd7bd4700 1 -- 172.21.15.66:0/3065665150 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f9cd0045ad0 con 0x7f9cd0060280 2023-11-17T05:33:27.955 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:27.952+0000 7f9cc27fc700 1 --2- 172.21.15.66:0/3065665150 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f9cbc02ff60 0x7f9cbc032420 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:33:27.955 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:27.952+0000 7f9cc27fc700 1 -- 172.21.15.66:0/3065665150 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(5..5 src has 1..5) v4 ==== 1270+0+0 (secure 0 0 0) 0x7f9ccc041000 con 0x7f9cd0060280 2023-11-17T05:33:27.955 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:27.952+0000 7f9cd516f700 1 --2- 172.21.15.66:0/3065665150 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f9cbc02ff60 0x7f9cbc032420 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:33:27.955 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:27.952+0000 7f9cd516f700 1 --2- 172.21.15.66:0/3065665150 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f9cbc02ff60 0x7f9cbc032420 secure :-1 s=READY pgs=152 cs=0 l=1 rev1=1 rx=0x7f9cc40096a0 tx=0x7f9cc4006b40).ready entity=mgr.14164 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:33:27.958 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:27.957+0000 7f9cc27fc700 1 -- 172.21.15.66:0/3065665150 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+152212 (secure 0 0 0) 0x7f9ccc01c790 con 0x7f9cd0060280 2023-11-17T05:33:27.991 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:33:27 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:33:27.535401+0000 mgr.smithi040.ogcprl (mgr.14164) 379 : cluster [DBG] pgmap v363: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:33:28.182 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:28.180+0000 7f9cd7bd4700 1 -- 172.21.15.66:0/3065665150 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "mon dump", "format": "json"} v 0) v1 -- 0x7f9cd0108110 con 0x7f9cd0060280 2023-11-17T05:33:28.182 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:28.181+0000 7f9cc27fc700 1 -- 172.21.15.66:0/3065665150 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mon_command_ack([{"prefix": "mon dump", "format": "json"}]=0 dumped monmap epoch 1 v1) v1 ==== 95+0+690 (secure 0 0 0) 0x7f9ccc010230 con 0x7f9cd0060280 2023-11-17T05:33:28.183 INFO:teuthology.orchestra.run.smithi066.stdout: 2023-11-17T05:33:28.183 INFO:teuthology.orchestra.run.smithi066.stdout:{"epoch":1,"fsid":"ed6470e2-8508-11ee-8dbe-212e2dc638e7","modified":"2023-11-17T05:20:13.934038Z","created":"2023-11-17T05:20:13.934038Z","min_mon_release":16,"min_mon_release_name":"pacific","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging"],"optional":[]},"mons":[{"rank":0,"name":"smithi040","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:3300","nonce":0},{"type":"v1","addr":"172.21.15.40:6789","nonce":0}]},"addr":"172.21.15.40:6789/0","public_addr":"172.21.15.40:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2023-11-17T05:33:28.184 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:28.183+0000 7f9cd7bd4700 1 -- 172.21.15.66:0/3065665150 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f9cbc02ff60 msgr2=0x7f9cbc032420 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:33:28.184 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:28.183+0000 7f9cd7bd4700 1 --2- 172.21.15.66:0/3065665150 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f9cbc02ff60 0x7f9cbc032420 secure :-1 s=READY pgs=152 cs=0 l=1 rev1=1 rx=0x7f9cc40096a0 tx=0x7f9cc4006b40).stop 2023-11-17T05:33:28.185 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:28.183+0000 7f9cd7bd4700 1 -- 172.21.15.66:0/3065665150 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f9cd0060280 msgr2=0x7f9cd01066e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:33:28.185 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:28.183+0000 7f9cd7bd4700 1 --2- 172.21.15.66:0/3065665150 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f9cd0060280 0x7f9cd01066e0 secure :-1 s=READY pgs=374 cs=0 l=1 rev1=1 rx=0x7f9ccc013510 tx=0x7f9ccc00b040).stop 2023-11-17T05:33:28.185 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:28.183+0000 7f9cd7bd4700 1 -- 172.21.15.66:0/3065665150 shutdown_connections 2023-11-17T05:33:28.185 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:28.183+0000 7f9cd7bd4700 1 --2- 172.21.15.66:0/3065665150 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f9cbc02ff60 0x7f9cbc032420 unknown :-1 s=CLOSED pgs=152 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:33:28.185 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:28.183+0000 7f9cd7bd4700 1 --2- 172.21.15.66:0/3065665150 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f9cd0060280 0x7f9cd01066e0 unknown :-1 s=CLOSED pgs=374 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:33:28.185 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:28.183+0000 7f9cd7bd4700 1 -- 172.21.15.66:0/3065665150 >> 172.21.15.66:0/3065665150 conn(0x7f9cd00f0b70 msgr2=0x7f9cd00f1660 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:33:28.185 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:28.183+0000 7f9cd7bd4700 1 -- 172.21.15.66:0/3065665150 shutdown_connections 2023-11-17T05:33:28.185 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:28.183+0000 7f9cd7bd4700 1 -- 172.21.15.66:0/3065665150 wait complete. 2023-11-17T05:33:28.185 INFO:teuthology.orchestra.run.smithi066.stderr:dumped monmap epoch 1 2023-11-17T05:33:28.988 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:33:28 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:33:28.182832+0000 mon.smithi040 (mon.0) 301 : audit [DBG] from='client.? 172.21.15.66:0/3065665150' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-11-17T05:33:29.591 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2023-11-17T05:33:29.591 DEBUG:teuthology.orchestra.run.smithi066:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.4 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ed6470e2-8508-11ee-8dbe-212e2dc638e7 -- ceph mon dump -f json 2023-11-17T05:33:30.757 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:30.754+0000 7f303d717700 1 -- 172.21.15.66:0/3518390563 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f30380f51a0 msgr2=0x7f30380f55c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:33:30.757 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:30.754+0000 7f303d717700 1 --2- 172.21.15.66:0/3518390563 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f30380f51a0 0x7f30380f55c0 secure :-1 s=READY pgs=375 cs=0 l=1 rev1=1 rx=0x7f3028009fb0 tx=0x7f3028009b30).stop 2023-11-17T05:33:30.757 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:30.754+0000 7f303d717700 1 -- 172.21.15.66:0/3518390563 shutdown_connections 2023-11-17T05:33:30.757 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:30.754+0000 7f303d717700 1 --2- 172.21.15.66:0/3518390563 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f30380f51a0 0x7f30380f55c0 unknown :-1 s=CLOSED pgs=375 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:33:30.757 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:30.754+0000 7f303d717700 1 -- 172.21.15.66:0/3518390563 >> 172.21.15.66:0/3518390563 conn(0x7f30380f09a0 msgr2=0x7f30380f2dc0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:33:30.757 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:30.754+0000 7f303d717700 1 -- 172.21.15.66:0/3518390563 shutdown_connections 2023-11-17T05:33:30.757 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:30.754+0000 7f303d717700 1 -- 172.21.15.66:0/3518390563 wait complete. 2023-11-17T05:33:30.758 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:30.754+0000 7f303d717700 1 Processor -- start 2023-11-17T05:33:30.758 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:30.755+0000 7f303d717700 1 -- start start 2023-11-17T05:33:30.758 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:30.755+0000 7f303d717700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f30380f51a0 0x7f30381091b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:33:30.758 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:30.755+0000 7f303d717700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f3038060240 con 0x7f30380f51a0 2023-11-17T05:33:30.758 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:30.755+0000 7f3037fff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f30380f51a0 0x7f30381091b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:33:30.758 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:30.755+0000 7f3037fff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f30380f51a0 0x7f30381091b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.66:60324/0 (socket says 172.21.15.66:60324) 2023-11-17T05:33:30.758 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:30.755+0000 7f3037fff700 1 -- 172.21.15.66:0/3499323578 learned_addr learned my addr 172.21.15.66:0/3499323578 (peer_addr_for_me v2:172.21.15.66:0/0) 2023-11-17T05:33:30.758 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:30.755+0000 7f3037fff700 1 -- 172.21.15.66:0/3499323578 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f3028005040 con 0x7f30380f51a0 2023-11-17T05:33:30.758 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:30.756+0000 7f3037fff700 1 --2- 172.21.15.66:0/3499323578 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f30380f51a0 0x7f30381091b0 secure :-1 s=READY pgs=376 cs=0 l=1 rev1=1 rx=0x7f3028013510 tx=0x7f302800b040).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:33:30.758 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:30.756+0000 7f3034ff9700 1 -- 172.21.15.66:0/3499323578 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f3028009320 con 0x7f30380f51a0 2023-11-17T05:33:30.759 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:30.756+0000 7f303d717700 1 -- 172.21.15.66:0/3499323578 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f3038060440 con 0x7f30380f51a0 2023-11-17T05:33:30.759 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:30.756+0000 7f3034ff9700 1 -- 172.21.15.66:0/3499323578 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(25 keys) v1 ==== 1074+0+0 (secure 0 0 0) 0x7f3028013f00 con 0x7f30380f51a0 2023-11-17T05:33:30.760 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:30.756+0000 7f3034ff9700 1 -- 172.21.15.66:0/3499323578 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f3028014dc0 con 0x7f30380f51a0 2023-11-17T05:33:30.760 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:30.756+0000 7f303d717700 1 -- 172.21.15.66:0/3499323578 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f3038109860 con 0x7f30380f51a0 2023-11-17T05:33:30.760 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:30.757+0000 7f3034ff9700 1 -- 172.21.15.66:0/3499323578 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 17) v1 ==== 36269+0+0 (secure 0 0 0) 0x7f30280095f0 con 0x7f30380f51a0 2023-11-17T05:33:30.760 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:30.757+0000 7f303d717700 1 -- 172.21.15.66:0/3499323578 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f3038045ad0 con 0x7f30380f51a0 2023-11-17T05:33:30.763 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:30.757+0000 7f3034ff9700 1 --2- 172.21.15.66:0/3499323578 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f302002ff60 0x7f3020032420 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:33:30.763 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:30.757+0000 7f3034ff9700 1 -- 172.21.15.66:0/3499323578 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(5..5 src has 1..5) v4 ==== 1270+0+0 (secure 0 0 0) 0x7f3028041000 con 0x7f30380f51a0 2023-11-17T05:33:30.763 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:30.757+0000 7f30377fe700 1 --2- 172.21.15.66:0/3499323578 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f302002ff60 0x7f3020032420 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:33:30.764 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:30.758+0000 7f30377fe700 1 --2- 172.21.15.66:0/3499323578 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f302002ff60 0x7f3020032420 secure :-1 s=READY pgs=153 cs=0 l=1 rev1=1 rx=0x7f302c0096a0 tx=0x7f302c006b40).ready entity=mgr.14164 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:33:30.764 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:30.762+0000 7f3034ff9700 1 -- 172.21.15.66:0/3499323578 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+152212 (secure 0 0 0) 0x7f302801c790 con 0x7f30380f51a0 2023-11-17T05:33:30.980 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:30.979+0000 7f303d717700 1 -- 172.21.15.66:0/3499323578 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "mon dump", "format": "json"} v 0) v1 -- 0x7f303810a070 con 0x7f30380f51a0 2023-11-17T05:33:30.981 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:30.979+0000 7f3034ff9700 1 -- 172.21.15.66:0/3499323578 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mon_command_ack([{"prefix": "mon dump", "format": "json"}]=0 dumped monmap epoch 1 v1) v1 ==== 95+0+690 (secure 0 0 0) 0x7f302801c790 con 0x7f30380f51a0 2023-11-17T05:33:30.981 INFO:teuthology.orchestra.run.smithi066.stdout: 2023-11-17T05:33:30.981 INFO:teuthology.orchestra.run.smithi066.stdout:{"epoch":1,"fsid":"ed6470e2-8508-11ee-8dbe-212e2dc638e7","modified":"2023-11-17T05:20:13.934038Z","created":"2023-11-17T05:20:13.934038Z","min_mon_release":16,"min_mon_release_name":"pacific","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging"],"optional":[]},"mons":[{"rank":0,"name":"smithi040","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:3300","nonce":0},{"type":"v1","addr":"172.21.15.40:6789","nonce":0}]},"addr":"172.21.15.40:6789/0","public_addr":"172.21.15.40:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2023-11-17T05:33:30.982 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:30.981+0000 7f303d717700 1 -- 172.21.15.66:0/3499323578 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f302002ff60 msgr2=0x7f3020032420 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:33:30.982 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:30.981+0000 7f303d717700 1 --2- 172.21.15.66:0/3499323578 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f302002ff60 0x7f3020032420 secure :-1 s=READY pgs=153 cs=0 l=1 rev1=1 rx=0x7f302c0096a0 tx=0x7f302c006b40).stop 2023-11-17T05:33:30.982 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:30.981+0000 7f303d717700 1 -- 172.21.15.66:0/3499323578 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f30380f51a0 msgr2=0x7f30381091b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:33:30.983 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:30.981+0000 7f303d717700 1 --2- 172.21.15.66:0/3499323578 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f30380f51a0 0x7f30381091b0 secure :-1 s=READY pgs=376 cs=0 l=1 rev1=1 rx=0x7f3028013510 tx=0x7f302800b040).stop 2023-11-17T05:33:30.983 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:30.981+0000 7f303d717700 1 -- 172.21.15.66:0/3499323578 shutdown_connections 2023-11-17T05:33:30.983 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:30.981+0000 7f303d717700 1 --2- 172.21.15.66:0/3499323578 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f302002ff60 0x7f3020032420 unknown :-1 s=CLOSED pgs=153 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:33:30.983 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:30.981+0000 7f303d717700 1 --2- 172.21.15.66:0/3499323578 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f30380f51a0 0x7f30381091b0 unknown :-1 s=CLOSED pgs=376 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:33:30.983 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:30.981+0000 7f303d717700 1 -- 172.21.15.66:0/3499323578 >> 172.21.15.66:0/3499323578 conn(0x7f30380f09a0 msgr2=0x7f30380f1460 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:33:30.983 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:30.981+0000 7f303d717700 1 -- 172.21.15.66:0/3499323578 shutdown_connections 2023-11-17T05:33:30.983 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:30.981+0000 7f303d717700 1 -- 172.21.15.66:0/3499323578 wait complete. 2023-11-17T05:33:30.983 INFO:teuthology.orchestra.run.smithi066.stderr:dumped monmap epoch 1 2023-11-17T05:33:30.988 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:33:30 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:33:29.535785+0000 mgr.smithi040.ogcprl (mgr.14164) 380 : cluster [DBG] pgmap v364: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:33:31.988 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:33:31 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:33:30.981224+0000 mon.smithi040 (mon.0) 302 : audit [DBG] from='client.? 172.21.15.66:0/3499323578' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-11-17T05:33:32.383 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2023-11-17T05:33:32.383 DEBUG:teuthology.orchestra.run.smithi066:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.4 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ed6470e2-8508-11ee-8dbe-212e2dc638e7 -- ceph mon dump -f json 2023-11-17T05:33:32.988 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:33:32 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:33:31.536065+0000 mgr.smithi040.ogcprl (mgr.14164) 381 : cluster [DBG] pgmap v365: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:33:33.570 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:33.567+0000 7f05338d9700 1 -- 172.21.15.66:0/3469180589 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f052c0f5410 msgr2=0x7f052c0f5830 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:33:33.570 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:33.567+0000 7f05338d9700 1 --2- 172.21.15.66:0/3469180589 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f052c0f5410 0x7f052c0f5830 secure :-1 s=READY pgs=377 cs=0 l=1 rev1=1 rx=0x7f0528009fb0 tx=0x7f0528009b30).stop 2023-11-17T05:33:33.571 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:33.567+0000 7f05338d9700 1 -- 172.21.15.66:0/3469180589 shutdown_connections 2023-11-17T05:33:33.571 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:33.567+0000 7f05338d9700 1 --2- 172.21.15.66:0/3469180589 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f052c0f5410 0x7f052c0f5830 unknown :-1 s=CLOSED pgs=377 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:33:33.571 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:33.567+0000 7f05338d9700 1 -- 172.21.15.66:0/3469180589 >> 172.21.15.66:0/3469180589 conn(0x7f052c0f0b70 msgr2=0x7f052c0f2f90 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:33:33.571 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:33.567+0000 7f05338d9700 1 -- 172.21.15.66:0/3469180589 shutdown_connections 2023-11-17T05:33:33.571 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:33.568+0000 7f05338d9700 1 -- 172.21.15.66:0/3469180589 wait complete. 2023-11-17T05:33:33.571 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:33.568+0000 7f05338d9700 1 Processor -- start 2023-11-17T05:33:33.571 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:33.568+0000 7f05338d9700 1 -- start start 2023-11-17T05:33:33.571 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:33.568+0000 7f05338d9700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f052c0f5410 0x7f052c106730 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:33:33.571 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:33.568+0000 7f05338d9700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f052c106c20 con 0x7f052c0f5410 2023-11-17T05:33:33.571 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:33.568+0000 7f0531675700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f052c0f5410 0x7f052c106730 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:33:33.571 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:33.568+0000 7f0531675700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f052c0f5410 0x7f052c106730 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.66:50474/0 (socket says 172.21.15.66:50474) 2023-11-17T05:33:33.571 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:33.568+0000 7f0531675700 1 -- 172.21.15.66:0/1060735579 learned_addr learned my addr 172.21.15.66:0/1060735579 (peer_addr_for_me v2:172.21.15.66:0/0) 2023-11-17T05:33:33.571 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:33.569+0000 7f0531675700 1 -- 172.21.15.66:0/1060735579 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f0528005040 con 0x7f052c0f5410 2023-11-17T05:33:33.572 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:33.569+0000 7f0531675700 1 --2- 172.21.15.66:0/1060735579 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f052c0f5410 0x7f052c106730 secure :-1 s=READY pgs=378 cs=0 l=1 rev1=1 rx=0x7f0528013510 tx=0x7f052800b040).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:33:33.572 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:33.569+0000 7f0521ffb700 1 -- 172.21.15.66:0/1060735579 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f0528009320 con 0x7f052c0f5410 2023-11-17T05:33:33.572 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:33.569+0000 7f05338d9700 1 -- 172.21.15.66:0/1060735579 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f052c106e20 con 0x7f052c0f5410 2023-11-17T05:33:33.572 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:33.569+0000 7f0521ffb700 1 -- 172.21.15.66:0/1060735579 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(25 keys) v1 ==== 1074+0+0 (secure 0 0 0) 0x7f0528013f00 con 0x7f052c0f5410 2023-11-17T05:33:33.572 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:33.569+0000 7f0521ffb700 1 -- 172.21.15.66:0/1060735579 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f0528014d90 con 0x7f052c0f5410 2023-11-17T05:33:33.572 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:33.569+0000 7f05338d9700 1 -- 172.21.15.66:0/1060735579 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f052c1072b0 con 0x7f052c0f5410 2023-11-17T05:33:33.573 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:33.570+0000 7f0521ffb700 1 -- 172.21.15.66:0/1060735579 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 17) v1 ==== 36269+0+0 (secure 0 0 0) 0x7f05280095f0 con 0x7f052c0f5410 2023-11-17T05:33:33.573 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:33.570+0000 7f05338d9700 1 -- 172.21.15.66:0/1060735579 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f052c045ad0 con 0x7f052c0f5410 2023-11-17T05:33:33.573 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:33.570+0000 7f0521ffb700 1 --2- 172.21.15.66:0/1060735579 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f051802ffb0 0x7f0518032470 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:33:33.573 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:33.570+0000 7f0521ffb700 1 -- 172.21.15.66:0/1060735579 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(5..5 src has 1..5) v4 ==== 1270+0+0 (secure 0 0 0) 0x7f0528040f90 con 0x7f052c0f5410 2023-11-17T05:33:33.573 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:33.571+0000 7f0530e74700 1 --2- 172.21.15.66:0/1060735579 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f051802ffb0 0x7f0518032470 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:33:33.573 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:33.571+0000 7f0530e74700 1 --2- 172.21.15.66:0/1060735579 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f051802ffb0 0x7f0518032470 secure :-1 s=READY pgs=154 cs=0 l=1 rev1=1 rx=0x7f051c0096a0 tx=0x7f051c006b40).ready entity=mgr.14164 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:33:33.576 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:33.575+0000 7f0521ffb700 1 -- 172.21.15.66:0/1060735579 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+152212 (secure 0 0 0) 0x7f0528014420 con 0x7f052c0f5410 2023-11-17T05:33:33.796 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:33.794+0000 7f05338d9700 1 -- 172.21.15.66:0/1060735579 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "mon dump", "format": "json"} v 0) v1 -- 0x7f052c0fae60 con 0x7f052c0f5410 2023-11-17T05:33:33.796 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:33.795+0000 7f0521ffb700 1 -- 172.21.15.66:0/1060735579 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mon_command_ack([{"prefix": "mon dump", "format": "json"}]=0 dumped monmap epoch 1 v1) v1 ==== 95+0+690 (secure 0 0 0) 0x7f052801d7f0 con 0x7f052c0f5410 2023-11-17T05:33:33.797 INFO:teuthology.orchestra.run.smithi066.stdout: 2023-11-17T05:33:33.797 INFO:teuthology.orchestra.run.smithi066.stdout:{"epoch":1,"fsid":"ed6470e2-8508-11ee-8dbe-212e2dc638e7","modified":"2023-11-17T05:20:13.934038Z","created":"2023-11-17T05:20:13.934038Z","min_mon_release":16,"min_mon_release_name":"pacific","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging"],"optional":[]},"mons":[{"rank":0,"name":"smithi040","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:3300","nonce":0},{"type":"v1","addr":"172.21.15.40:6789","nonce":0}]},"addr":"172.21.15.40:6789/0","public_addr":"172.21.15.40:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2023-11-17T05:33:33.799 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:33.797+0000 7f05338d9700 1 -- 172.21.15.66:0/1060735579 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f051802ffb0 msgr2=0x7f0518032470 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:33:33.799 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:33.797+0000 7f05338d9700 1 --2- 172.21.15.66:0/1060735579 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f051802ffb0 0x7f0518032470 secure :-1 s=READY pgs=154 cs=0 l=1 rev1=1 rx=0x7f051c0096a0 tx=0x7f051c006b40).stop 2023-11-17T05:33:33.799 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:33.797+0000 7f05338d9700 1 -- 172.21.15.66:0/1060735579 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f052c0f5410 msgr2=0x7f052c106730 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:33:33.799 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:33.797+0000 7f05338d9700 1 --2- 172.21.15.66:0/1060735579 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f052c0f5410 0x7f052c106730 secure :-1 s=READY pgs=378 cs=0 l=1 rev1=1 rx=0x7f0528013510 tx=0x7f052800b040).stop 2023-11-17T05:33:33.799 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:33.797+0000 7f05338d9700 1 -- 172.21.15.66:0/1060735579 shutdown_connections 2023-11-17T05:33:33.799 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:33.797+0000 7f05338d9700 1 --2- 172.21.15.66:0/1060735579 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f051802ffb0 0x7f0518032470 unknown :-1 s=CLOSED pgs=154 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:33:33.799 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:33.797+0000 7f05338d9700 1 --2- 172.21.15.66:0/1060735579 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f052c0f5410 0x7f052c106730 unknown :-1 s=CLOSED pgs=378 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:33:33.799 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:33.797+0000 7f05338d9700 1 -- 172.21.15.66:0/1060735579 >> 172.21.15.66:0/1060735579 conn(0x7f052c0f0b70 msgr2=0x7f052c0f1660 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:33:33.799 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:33.797+0000 7f05338d9700 1 -- 172.21.15.66:0/1060735579 shutdown_connections 2023-11-17T05:33:33.799 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:33.797+0000 7f05338d9700 1 -- 172.21.15.66:0/1060735579 wait complete. 2023-11-17T05:33:33.799 INFO:teuthology.orchestra.run.smithi066.stderr:dumped monmap epoch 1 2023-11-17T05:33:33.988 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:33:33 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:33:33.536369+0000 mgr.smithi040.ogcprl (mgr.14164) 382 : cluster [DBG] pgmap v366: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:33:34.988 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:33:34 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:33:33.796916+0000 mon.smithi040 (mon.0) 303 : audit [DBG] from='client.? 172.21.15.66:0/1060735579' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-11-17T05:33:35.146 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2023-11-17T05:33:35.146 DEBUG:teuthology.orchestra.run.smithi066:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.4 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ed6470e2-8508-11ee-8dbe-212e2dc638e7 -- ceph mon dump -f json 2023-11-17T05:33:35.988 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:33:35 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:33:35.536784+0000 mgr.smithi040.ogcprl (mgr.14164) 383 : cluster [DBG] pgmap v367: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:33:36.384 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:36.377+0000 7f785d9b1700 1 -- 172.21.15.66:0/1373740101 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f78580f4740 msgr2=0x7f78580f4b60 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:33:36.384 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:36.377+0000 7f785d9b1700 1 --2- 172.21.15.66:0/1373740101 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f78580f4740 0x7f78580f4b60 secure :-1 s=READY pgs=379 cs=0 l=1 rev1=1 rx=0x7f7848009fb0 tx=0x7f7848009b30).stop 2023-11-17T05:33:36.385 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:36.377+0000 7f785d9b1700 1 -- 172.21.15.66:0/1373740101 shutdown_connections 2023-11-17T05:33:36.385 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:36.377+0000 7f785d9b1700 1 --2- 172.21.15.66:0/1373740101 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f78580f4740 0x7f78580f4b60 unknown :-1 s=CLOSED pgs=379 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:33:36.385 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:36.377+0000 7f785d9b1700 1 -- 172.21.15.66:0/1373740101 >> 172.21.15.66:0/1373740101 conn(0x7f78580f0260 msgr2=0x7f78580f2680 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:33:36.385 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:36.377+0000 7f785d9b1700 1 -- 172.21.15.66:0/1373740101 shutdown_connections 2023-11-17T05:33:36.385 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:36.377+0000 7f785d9b1700 1 -- 172.21.15.66:0/1373740101 wait complete. 2023-11-17T05:33:36.385 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:36.378+0000 7f785d9b1700 1 Processor -- start 2023-11-17T05:33:36.385 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:36.378+0000 7f785d9b1700 1 -- start start 2023-11-17T05:33:36.385 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:36.378+0000 7f785d9b1700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f78580f4740 0x7f78581088f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:33:36.385 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:36.378+0000 7f785d9b1700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f7858108de0 con 0x7f78580f4740 2023-11-17T05:33:36.385 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:36.379+0000 7f7856ffd700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f78580f4740 0x7f78581088f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:33:36.385 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:36.379+0000 7f7856ffd700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f78580f4740 0x7f78581088f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.66:50492/0 (socket says 172.21.15.66:50492) 2023-11-17T05:33:36.385 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:36.379+0000 7f7856ffd700 1 -- 172.21.15.66:0/3877224388 learned_addr learned my addr 172.21.15.66:0/3877224388 (peer_addr_for_me v2:172.21.15.66:0/0) 2023-11-17T05:33:36.386 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:36.379+0000 7f7856ffd700 1 -- 172.21.15.66:0/3877224388 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f7848005040 con 0x7f78580f4740 2023-11-17T05:33:36.386 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:36.379+0000 7f7856ffd700 1 --2- 172.21.15.66:0/3877224388 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f78580f4740 0x7f78581088f0 secure :-1 s=READY pgs=380 cs=0 l=1 rev1=1 rx=0x7f7848013510 tx=0x7f784800b040).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:33:36.386 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:36.379+0000 7f783f7fe700 1 -- 172.21.15.66:0/3877224388 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f7848009320 con 0x7f78580f4740 2023-11-17T05:33:36.386 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:36.379+0000 7f783f7fe700 1 -- 172.21.15.66:0/3877224388 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(25 keys) v1 ==== 1074+0+0 (secure 0 0 0) 0x7f7848013f00 con 0x7f78580f4740 2023-11-17T05:33:36.386 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:36.379+0000 7f785d9b1700 1 -- 172.21.15.66:0/3877224388 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f7858108fe0 con 0x7f78580f4740 2023-11-17T05:33:36.386 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:36.379+0000 7f783f7fe700 1 -- 172.21.15.66:0/3877224388 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f7848014d90 con 0x7f78580f4740 2023-11-17T05:33:36.386 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:36.379+0000 7f785d9b1700 1 -- 172.21.15.66:0/3877224388 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f7858109470 con 0x7f78580f4740 2023-11-17T05:33:36.386 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:36.380+0000 7f783f7fe700 1 -- 172.21.15.66:0/3877224388 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 17) v1 ==== 36269+0+0 (secure 0 0 0) 0x7f78480095f0 con 0x7f78580f4740 2023-11-17T05:33:36.386 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:36.380+0000 7f785d9b1700 1 -- 172.21.15.66:0/3877224388 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f7858045ad0 con 0x7f78580f4740 2023-11-17T05:33:36.386 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:36.380+0000 7f783f7fe700 1 --2- 172.21.15.66:0/3877224388 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f784002ffb0 0x7f7840032470 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:33:36.387 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:36.380+0000 7f783f7fe700 1 -- 172.21.15.66:0/3877224388 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(5..5 src has 1..5) v4 ==== 1270+0+0 (secure 0 0 0) 0x7f7848040fc0 con 0x7f78580f4740 2023-11-17T05:33:36.387 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:36.381+0000 7f78567fc700 1 --2- 172.21.15.66:0/3877224388 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f784002ffb0 0x7f7840032470 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:33:36.387 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:36.381+0000 7f78567fc700 1 --2- 172.21.15.66:0/3877224388 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f784002ffb0 0x7f7840032470 secure :-1 s=READY pgs=155 cs=0 l=1 rev1=1 rx=0x7f784c0096a0 tx=0x7f784c006b40).ready entity=mgr.14164 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:33:36.387 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:36.385+0000 7f783f7fe700 1 -- 172.21.15.66:0/3877224388 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+152212 (secure 0 0 0) 0x7f7848014420 con 0x7f78580f4740 2023-11-17T05:33:36.604 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:36.603+0000 7f785d9b1700 1 -- 172.21.15.66:0/3877224388 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "mon dump", "format": "json"} v 0) v1 -- 0x7f785810a320 con 0x7f78580f4740 2023-11-17T05:33:36.606 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:36.604+0000 7f783f7fe700 1 -- 172.21.15.66:0/3877224388 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mon_command_ack([{"prefix": "mon dump", "format": "json"}]=0 dumped monmap epoch 1 v1) v1 ==== 95+0+690 (secure 0 0 0) 0x7f784801d7f0 con 0x7f78580f4740 2023-11-17T05:33:36.606 INFO:teuthology.orchestra.run.smithi066.stdout: 2023-11-17T05:33:36.606 INFO:teuthology.orchestra.run.smithi066.stdout:{"epoch":1,"fsid":"ed6470e2-8508-11ee-8dbe-212e2dc638e7","modified":"2023-11-17T05:20:13.934038Z","created":"2023-11-17T05:20:13.934038Z","min_mon_release":16,"min_mon_release_name":"pacific","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging"],"optional":[]},"mons":[{"rank":0,"name":"smithi040","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:3300","nonce":0},{"type":"v1","addr":"172.21.15.40:6789","nonce":0}]},"addr":"172.21.15.40:6789/0","public_addr":"172.21.15.40:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2023-11-17T05:33:36.607 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:36.606+0000 7f785d9b1700 1 -- 172.21.15.66:0/3877224388 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f784002ffb0 msgr2=0x7f7840032470 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:33:36.607 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:36.606+0000 7f785d9b1700 1 --2- 172.21.15.66:0/3877224388 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f784002ffb0 0x7f7840032470 secure :-1 s=READY pgs=155 cs=0 l=1 rev1=1 rx=0x7f784c0096a0 tx=0x7f784c006b40).stop 2023-11-17T05:33:36.607 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:36.606+0000 7f785d9b1700 1 -- 172.21.15.66:0/3877224388 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f78580f4740 msgr2=0x7f78581088f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:33:36.607 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:36.606+0000 7f785d9b1700 1 --2- 172.21.15.66:0/3877224388 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f78580f4740 0x7f78581088f0 secure :-1 s=READY pgs=380 cs=0 l=1 rev1=1 rx=0x7f7848013510 tx=0x7f784800b040).stop 2023-11-17T05:33:36.607 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:36.606+0000 7f785d9b1700 1 -- 172.21.15.66:0/3877224388 shutdown_connections 2023-11-17T05:33:36.607 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:36.606+0000 7f785d9b1700 1 --2- 172.21.15.66:0/3877224388 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f784002ffb0 0x7f7840032470 unknown :-1 s=CLOSED pgs=155 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:33:36.608 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:36.606+0000 7f785d9b1700 1 --2- 172.21.15.66:0/3877224388 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f78580f4740 0x7f78581088f0 unknown :-1 s=CLOSED pgs=380 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:33:36.608 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:36.606+0000 7f785d9b1700 1 -- 172.21.15.66:0/3877224388 >> 172.21.15.66:0/3877224388 conn(0x7f78580f0260 msgr2=0x7f78580f0d40 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:33:36.608 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:36.606+0000 7f785d9b1700 1 -- 172.21.15.66:0/3877224388 shutdown_connections 2023-11-17T05:33:36.608 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:36.606+0000 7f785d9b1700 1 -- 172.21.15.66:0/3877224388 wait complete. 2023-11-17T05:33:36.608 INFO:teuthology.orchestra.run.smithi066.stderr:dumped monmap epoch 1 2023-11-17T05:33:36.988 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:33:36 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:33:36.605722+0000 mon.smithi040 (mon.0) 304 : audit [DBG] from='client.? 172.21.15.66:0/3877224388' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-11-17T05:33:37.989 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:33:37 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:33:37.537150+0000 mgr.smithi040.ogcprl (mgr.14164) 384 : cluster [DBG] pgmap v368: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:33:38.014 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2023-11-17T05:33:38.014 DEBUG:teuthology.orchestra.run.smithi066:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.4 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ed6470e2-8508-11ee-8dbe-212e2dc638e7 -- ceph mon dump -f json 2023-11-17T05:33:39.234 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:39.232+0000 7f6e25a5f700 1 -- 172.21.15.66:0/3414043264 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f6e200f6510 msgr2=0x7f6e200f6930 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:33:39.234 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:39.232+0000 7f6e25a5f700 1 --2- 172.21.15.66:0/3414043264 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f6e200f6510 0x7f6e200f6930 secure :-1 s=READY pgs=381 cs=0 l=1 rev1=1 rx=0x7f6e10009fb0 tx=0x7f6e10009b30).stop 2023-11-17T05:33:39.234 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:39.232+0000 7f6e25a5f700 1 -- 172.21.15.66:0/3414043264 shutdown_connections 2023-11-17T05:33:39.234 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:39.232+0000 7f6e25a5f700 1 --2- 172.21.15.66:0/3414043264 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f6e200f6510 0x7f6e200f6930 unknown :-1 s=CLOSED pgs=381 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:33:39.234 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:39.232+0000 7f6e25a5f700 1 -- 172.21.15.66:0/3414043264 >> 172.21.15.66:0/3414043264 conn(0x7f6e200f1c70 msgr2=0x7f6e200f4090 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:33:39.235 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:39.232+0000 7f6e25a5f700 1 -- 172.21.15.66:0/3414043264 shutdown_connections 2023-11-17T05:33:39.235 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:39.232+0000 7f6e25a5f700 1 -- 172.21.15.66:0/3414043264 wait complete. 2023-11-17T05:33:39.236 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:39.233+0000 7f6e25a5f700 1 Processor -- start 2023-11-17T05:33:39.236 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:39.233+0000 7f6e25a5f700 1 -- start start 2023-11-17T05:33:39.236 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:39.234+0000 7f6e25a5f700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f6e200f6510 0x7f6e20106720 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:33:39.236 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:39.234+0000 7f6e25a5f700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f6e20106c10 con 0x7f6e200f6510 2023-11-17T05:33:39.236 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:39.234+0000 7f6e1effd700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f6e200f6510 0x7f6e20106720 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:33:39.237 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:39.234+0000 7f6e1effd700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f6e200f6510 0x7f6e20106720 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.66:50512/0 (socket says 172.21.15.66:50512) 2023-11-17T05:33:39.237 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:39.234+0000 7f6e1effd700 1 -- 172.21.15.66:0/3087081747 learned_addr learned my addr 172.21.15.66:0/3087081747 (peer_addr_for_me v2:172.21.15.66:0/0) 2023-11-17T05:33:39.237 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:39.234+0000 7f6e1effd700 1 -- 172.21.15.66:0/3087081747 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f6e10005040 con 0x7f6e200f6510 2023-11-17T05:33:39.237 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:39.234+0000 7f6e1effd700 1 --2- 172.21.15.66:0/3087081747 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f6e200f6510 0x7f6e20106720 secure :-1 s=READY pgs=382 cs=0 l=1 rev1=1 rx=0x7f6e10013510 tx=0x7f6e1000b040).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:33:39.237 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:39.234+0000 7f6e077fe700 1 -- 172.21.15.66:0/3087081747 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f6e10009320 con 0x7f6e200f6510 2023-11-17T05:33:39.237 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:39.234+0000 7f6e25a5f700 1 -- 172.21.15.66:0/3087081747 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f6e20106e10 con 0x7f6e200f6510 2023-11-17T05:33:39.237 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:39.234+0000 7f6e077fe700 1 -- 172.21.15.66:0/3087081747 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(25 keys) v1 ==== 1074+0+0 (secure 0 0 0) 0x7f6e10013f00 con 0x7f6e200f6510 2023-11-17T05:33:39.237 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:39.235+0000 7f6e077fe700 1 -- 172.21.15.66:0/3087081747 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f6e10014d90 con 0x7f6e200f6510 2023-11-17T05:33:39.237 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:39.235+0000 7f6e25a5f700 1 -- 172.21.15.66:0/3087081747 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f6e201072a0 con 0x7f6e200f6510 2023-11-17T05:33:39.238 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:39.235+0000 7f6e077fe700 1 -- 172.21.15.66:0/3087081747 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 17) v1 ==== 36269+0+0 (secure 0 0 0) 0x7f6e100095f0 con 0x7f6e200f6510 2023-11-17T05:33:39.238 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:39.235+0000 7f6e25a5f700 1 -- 172.21.15.66:0/3087081747 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f6e20045ad0 con 0x7f6e200f6510 2023-11-17T05:33:39.238 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:39.236+0000 7f6e077fe700 1 --2- 172.21.15.66:0/3087081747 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f6e0802ff60 0x7f6e08032420 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:33:39.238 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:39.236+0000 7f6e077fe700 1 -- 172.21.15.66:0/3087081747 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(5..5 src has 1..5) v4 ==== 1270+0+0 (secure 0 0 0) 0x7f6e10041000 con 0x7f6e200f6510 2023-11-17T05:33:39.238 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:39.236+0000 7f6e1e7fc700 1 --2- 172.21.15.66:0/3087081747 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f6e0802ff60 0x7f6e08032420 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:33:39.238 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:39.236+0000 7f6e1e7fc700 1 --2- 172.21.15.66:0/3087081747 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f6e0802ff60 0x7f6e08032420 secure :-1 s=READY pgs=156 cs=0 l=1 rev1=1 rx=0x7f6e140096a0 tx=0x7f6e14006b40).ready entity=mgr.14164 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:33:39.242 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:39.241+0000 7f6e077fe700 1 -- 172.21.15.66:0/3087081747 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+152212 (secure 0 0 0) 0x7f6e10014420 con 0x7f6e200f6510 2023-11-17T05:33:39.461 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:39.459+0000 7f6e25a5f700 1 -- 172.21.15.66:0/3087081747 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "mon dump", "format": "json"} v 0) v1 -- 0x7f6e20108150 con 0x7f6e200f6510 2023-11-17T05:33:39.461 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:39.460+0000 7f6e077fe700 1 -- 172.21.15.66:0/3087081747 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mon_command_ack([{"prefix": "mon dump", "format": "json"}]=0 dumped monmap epoch 1 v1) v1 ==== 95+0+690 (secure 0 0 0) 0x7f6e10010230 con 0x7f6e200f6510 2023-11-17T05:33:39.461 INFO:teuthology.orchestra.run.smithi066.stdout: 2023-11-17T05:33:39.461 INFO:teuthology.orchestra.run.smithi066.stdout:{"epoch":1,"fsid":"ed6470e2-8508-11ee-8dbe-212e2dc638e7","modified":"2023-11-17T05:20:13.934038Z","created":"2023-11-17T05:20:13.934038Z","min_mon_release":16,"min_mon_release_name":"pacific","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging"],"optional":[]},"mons":[{"rank":0,"name":"smithi040","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:3300","nonce":0},{"type":"v1","addr":"172.21.15.40:6789","nonce":0}]},"addr":"172.21.15.40:6789/0","public_addr":"172.21.15.40:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2023-11-17T05:33:39.463 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:39.461+0000 7f6e25a5f700 1 -- 172.21.15.66:0/3087081747 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f6e0802ff60 msgr2=0x7f6e08032420 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:33:39.463 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:39.461+0000 7f6e25a5f700 1 --2- 172.21.15.66:0/3087081747 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f6e0802ff60 0x7f6e08032420 secure :-1 s=READY pgs=156 cs=0 l=1 rev1=1 rx=0x7f6e140096a0 tx=0x7f6e14006b40).stop 2023-11-17T05:33:39.463 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:39.461+0000 7f6e25a5f700 1 -- 172.21.15.66:0/3087081747 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f6e200f6510 msgr2=0x7f6e20106720 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:33:39.463 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:39.461+0000 7f6e25a5f700 1 --2- 172.21.15.66:0/3087081747 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f6e200f6510 0x7f6e20106720 secure :-1 s=READY pgs=382 cs=0 l=1 rev1=1 rx=0x7f6e10013510 tx=0x7f6e1000b040).stop 2023-11-17T05:33:39.463 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:39.461+0000 7f6e25a5f700 1 -- 172.21.15.66:0/3087081747 shutdown_connections 2023-11-17T05:33:39.463 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:39.461+0000 7f6e25a5f700 1 --2- 172.21.15.66:0/3087081747 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f6e0802ff60 0x7f6e08032420 unknown :-1 s=CLOSED pgs=156 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:33:39.463 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:39.462+0000 7f6e25a5f700 1 --2- 172.21.15.66:0/3087081747 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f6e200f6510 0x7f6e20106720 unknown :-1 s=CLOSED pgs=382 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:33:39.463 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:39.462+0000 7f6e25a5f700 1 -- 172.21.15.66:0/3087081747 >> 172.21.15.66:0/3087081747 conn(0x7f6e200f1c70 msgr2=0x7f6e200f27d0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:33:39.464 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:39.462+0000 7f6e25a5f700 1 -- 172.21.15.66:0/3087081747 shutdown_connections 2023-11-17T05:33:39.464 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:39.462+0000 7f6e25a5f700 1 -- 172.21.15.66:0/3087081747 wait complete. 2023-11-17T05:33:39.464 INFO:teuthology.orchestra.run.smithi066.stderr:dumped monmap epoch 1 2023-11-17T05:33:39.989 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:33:39 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:33:39.461751+0000 mon.smithi040 (mon.0) 305 : audit [DBG] from='client.? 172.21.15.66:0/3087081747' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-11-17T05:33:40.855 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2023-11-17T05:33:40.856 DEBUG:teuthology.orchestra.run.smithi066:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.4 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ed6470e2-8508-11ee-8dbe-212e2dc638e7 -- ceph mon dump -f json 2023-11-17T05:33:40.990 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:33:40 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:33:39.537449+0000 mgr.smithi040.ogcprl (mgr.14164) 385 : cluster [DBG] pgmap v369: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:33:41.988 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:33:41 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:33:41.537808+0000 mgr.smithi040.ogcprl (mgr.14164) 386 : cluster [DBG] pgmap v370: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:33:41.993 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:41.991+0000 7fbb307cd700 1 -- 172.21.15.66:0/1573132813 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fbb280f6510 msgr2=0x7fbb280f6930 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:33:41.994 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:41.991+0000 7fbb307cd700 1 --2- 172.21.15.66:0/1573132813 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fbb280f6510 0x7fbb280f6930 secure :-1 s=READY pgs=383 cs=0 l=1 rev1=1 rx=0x7fbb24009fb0 tx=0x7fbb24009b30).stop 2023-11-17T05:33:41.994 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:41.991+0000 7fbb307cd700 1 -- 172.21.15.66:0/1573132813 shutdown_connections 2023-11-17T05:33:41.994 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:41.991+0000 7fbb307cd700 1 --2- 172.21.15.66:0/1573132813 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fbb280f6510 0x7fbb280f6930 unknown :-1 s=CLOSED pgs=383 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:33:41.994 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:41.991+0000 7fbb307cd700 1 -- 172.21.15.66:0/1573132813 >> 172.21.15.66:0/1573132813 conn(0x7fbb280f1c70 msgr2=0x7fbb280f4090 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:33:41.994 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:41.991+0000 7fbb307cd700 1 -- 172.21.15.66:0/1573132813 shutdown_connections 2023-11-17T05:33:41.994 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:41.991+0000 7fbb307cd700 1 -- 172.21.15.66:0/1573132813 wait complete. 2023-11-17T05:33:41.994 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:41.991+0000 7fbb307cd700 1 Processor -- start 2023-11-17T05:33:41.994 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:41.991+0000 7fbb307cd700 1 -- start start 2023-11-17T05:33:41.994 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:41.991+0000 7fbb307cd700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fbb280f6510 0x7fbb28106730 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:33:41.994 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:41.991+0000 7fbb307cd700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fbb28106c20 con 0x7fbb280f6510 2023-11-17T05:33:41.994 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:41.992+0000 7fbb2e569700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fbb280f6510 0x7fbb28106730 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:33:41.994 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:41.992+0000 7fbb2e569700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fbb280f6510 0x7fbb28106730 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.66:50536/0 (socket says 172.21.15.66:50536) 2023-11-17T05:33:41.995 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:41.992+0000 7fbb2e569700 1 -- 172.21.15.66:0/1097865189 learned_addr learned my addr 172.21.15.66:0/1097865189 (peer_addr_for_me v2:172.21.15.66:0/0) 2023-11-17T05:33:41.995 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:41.992+0000 7fbb2e569700 1 -- 172.21.15.66:0/1097865189 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fbb24005040 con 0x7fbb280f6510 2023-11-17T05:33:41.995 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:41.992+0000 7fbb2e569700 1 --2- 172.21.15.66:0/1097865189 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fbb280f6510 0x7fbb28106730 secure :-1 s=READY pgs=384 cs=0 l=1 rev1=1 rx=0x7fbb24013510 tx=0x7fbb2400b040).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:33:41.995 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:41.992+0000 7fbb1effd700 1 -- 172.21.15.66:0/1097865189 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7fbb24009320 con 0x7fbb280f6510 2023-11-17T05:33:41.995 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:41.992+0000 7fbb307cd700 1 -- 172.21.15.66:0/1097865189 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fbb28106e20 con 0x7fbb280f6510 2023-11-17T05:33:41.995 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:41.992+0000 7fbb1effd700 1 -- 172.21.15.66:0/1097865189 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(25 keys) v1 ==== 1074+0+0 (secure 0 0 0) 0x7fbb24013f00 con 0x7fbb280f6510 2023-11-17T05:33:41.995 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:41.992+0000 7fbb1effd700 1 -- 172.21.15.66:0/1097865189 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7fbb24014d90 con 0x7fbb280f6510 2023-11-17T05:33:41.995 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:41.992+0000 7fbb307cd700 1 -- 172.21.15.66:0/1097865189 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fbb281072b0 con 0x7fbb280f6510 2023-11-17T05:33:41.996 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:41.993+0000 7fbb1effd700 1 -- 172.21.15.66:0/1097865189 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 17) v1 ==== 36269+0+0 (secure 0 0 0) 0x7fbb240095f0 con 0x7fbb280f6510 2023-11-17T05:33:41.996 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:41.993+0000 7fbb307cd700 1 -- 172.21.15.66:0/1097865189 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fbb28045ad0 con 0x7fbb280f6510 2023-11-17T05:33:41.996 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:41.993+0000 7fbb1effd700 1 --2- 172.21.15.66:0/1097865189 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fbb1402ff60 0x7fbb14032420 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:33:41.996 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:41.993+0000 7fbb1effd700 1 -- 172.21.15.66:0/1097865189 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(5..5 src has 1..5) v4 ==== 1270+0+0 (secure 0 0 0) 0x7fbb24041080 con 0x7fbb280f6510 2023-11-17T05:33:41.996 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:41.993+0000 7fbb2dd68700 1 --2- 172.21.15.66:0/1097865189 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fbb1402ff60 0x7fbb14032420 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:33:41.996 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:41.994+0000 7fbb2dd68700 1 --2- 172.21.15.66:0/1097865189 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fbb1402ff60 0x7fbb14032420 secure :-1 s=READY pgs=157 cs=0 l=1 rev1=1 rx=0x7fbb180096a0 tx=0x7fbb18006b40).ready entity=mgr.14164 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:33:42.003 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:41.998+0000 7fbb1effd700 1 -- 172.21.15.66:0/1097865189 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+152212 (secure 0 0 0) 0x7fbb24014420 con 0x7fbb280f6510 2023-11-17T05:33:42.223 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:42.222+0000 7fbb307cd700 1 -- 172.21.15.66:0/1097865189 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "mon dump", "format": "json"} v 0) v1 -- 0x7fbb280fbf60 con 0x7fbb280f6510 2023-11-17T05:33:42.224 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:42.223+0000 7fbb1effd700 1 -- 172.21.15.66:0/1097865189 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mon_command_ack([{"prefix": "mon dump", "format": "json"}]=0 dumped monmap epoch 1 v1) v1 ==== 95+0+690 (secure 0 0 0) 0x7fbb24010230 con 0x7fbb280f6510 2023-11-17T05:33:42.224 INFO:teuthology.orchestra.run.smithi066.stdout: 2023-11-17T05:33:42.224 INFO:teuthology.orchestra.run.smithi066.stdout:{"epoch":1,"fsid":"ed6470e2-8508-11ee-8dbe-212e2dc638e7","modified":"2023-11-17T05:20:13.934038Z","created":"2023-11-17T05:20:13.934038Z","min_mon_release":16,"min_mon_release_name":"pacific","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging"],"optional":[]},"mons":[{"rank":0,"name":"smithi040","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:3300","nonce":0},{"type":"v1","addr":"172.21.15.40:6789","nonce":0}]},"addr":"172.21.15.40:6789/0","public_addr":"172.21.15.40:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2023-11-17T05:33:42.226 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:42.224+0000 7fbb307cd700 1 -- 172.21.15.66:0/1097865189 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fbb1402ff60 msgr2=0x7fbb14032420 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:33:42.226 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:42.224+0000 7fbb307cd700 1 --2- 172.21.15.66:0/1097865189 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fbb1402ff60 0x7fbb14032420 secure :-1 s=READY pgs=157 cs=0 l=1 rev1=1 rx=0x7fbb180096a0 tx=0x7fbb18006b40).stop 2023-11-17T05:33:42.226 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:42.224+0000 7fbb307cd700 1 -- 172.21.15.66:0/1097865189 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fbb280f6510 msgr2=0x7fbb28106730 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:33:42.226 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:42.224+0000 7fbb307cd700 1 --2- 172.21.15.66:0/1097865189 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fbb280f6510 0x7fbb28106730 secure :-1 s=READY pgs=384 cs=0 l=1 rev1=1 rx=0x7fbb24013510 tx=0x7fbb2400b040).stop 2023-11-17T05:33:42.226 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:42.224+0000 7fbb307cd700 1 -- 172.21.15.66:0/1097865189 shutdown_connections 2023-11-17T05:33:42.226 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:42.224+0000 7fbb307cd700 1 --2- 172.21.15.66:0/1097865189 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fbb1402ff60 0x7fbb14032420 unknown :-1 s=CLOSED pgs=157 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:33:42.226 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:42.224+0000 7fbb307cd700 1 --2- 172.21.15.66:0/1097865189 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fbb280f6510 0x7fbb28106730 unknown :-1 s=CLOSED pgs=384 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:33:42.226 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:42.224+0000 7fbb307cd700 1 -- 172.21.15.66:0/1097865189 >> 172.21.15.66:0/1097865189 conn(0x7fbb280f1c70 msgr2=0x7fbb280f2760 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:33:42.227 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:42.224+0000 7fbb307cd700 1 -- 172.21.15.66:0/1097865189 shutdown_connections 2023-11-17T05:33:42.227 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:42.225+0000 7fbb307cd700 1 -- 172.21.15.66:0/1097865189 wait complete. 2023-11-17T05:33:42.227 INFO:teuthology.orchestra.run.smithi066.stderr:dumped monmap epoch 1 2023-11-17T05:33:42.989 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:33:42 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:33:42.224416+0000 mon.smithi040 (mon.0) 306 : audit [DBG] from='client.? 172.21.15.66:0/1097865189' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-11-17T05:33:43.604 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2023-11-17T05:33:43.604 DEBUG:teuthology.orchestra.run.smithi066:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.4 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ed6470e2-8508-11ee-8dbe-212e2dc638e7 -- ceph mon dump -f json 2023-11-17T05:33:43.989 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:33:43 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:33:43.538073+0000 mgr.smithi040.ogcprl (mgr.14164) 387 : cluster [DBG] pgmap v371: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:33:44.792 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:44.790+0000 7fcdcad41700 1 -- 172.21.15.66:0/1793915335 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fcdc40f51a0 msgr2=0x7fcdc40f55c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:33:44.792 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:44.790+0000 7fcdcad41700 1 --2- 172.21.15.66:0/1793915335 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fcdc40f51a0 0x7fcdc40f55c0 secure :-1 s=READY pgs=385 cs=0 l=1 rev1=1 rx=0x7fcdc0009fb0 tx=0x7fcdc0009b30).stop 2023-11-17T05:33:44.792 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:44.790+0000 7fcdcad41700 1 -- 172.21.15.66:0/1793915335 shutdown_connections 2023-11-17T05:33:44.792 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:44.790+0000 7fcdcad41700 1 --2- 172.21.15.66:0/1793915335 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fcdc40f51a0 0x7fcdc40f55c0 unknown :-1 s=CLOSED pgs=385 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:33:44.792 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:44.790+0000 7fcdcad41700 1 -- 172.21.15.66:0/1793915335 >> 172.21.15.66:0/1793915335 conn(0x7fcdc40f09a0 msgr2=0x7fcdc40f2dc0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:33:44.792 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:44.790+0000 7fcdcad41700 1 -- 172.21.15.66:0/1793915335 shutdown_connections 2023-11-17T05:33:44.792 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:44.790+0000 7fcdcad41700 1 -- 172.21.15.66:0/1793915335 wait complete. 2023-11-17T05:33:44.793 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:44.790+0000 7fcdcad41700 1 Processor -- start 2023-11-17T05:33:44.793 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:44.790+0000 7fcdcad41700 1 -- start start 2023-11-17T05:33:44.793 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:44.791+0000 7fcdcad41700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fcdc40f51a0 0x7fcdc40fd120 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:33:44.793 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:44.791+0000 7fcdcad41700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fcdc40fb5a0 con 0x7fcdc40f51a0 2023-11-17T05:33:44.793 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:44.791+0000 7fcdc9d3f700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fcdc40f51a0 0x7fcdc40fd120 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:33:44.793 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:44.791+0000 7fcdc9d3f700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fcdc40f51a0 0x7fcdc40fd120 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.66:58682/0 (socket says 172.21.15.66:58682) 2023-11-17T05:33:44.793 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:44.791+0000 7fcdc9d3f700 1 -- 172.21.15.66:0/1965504515 learned_addr learned my addr 172.21.15.66:0/1965504515 (peer_addr_for_me v2:172.21.15.66:0/0) 2023-11-17T05:33:44.793 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:44.791+0000 7fcdc9d3f700 1 -- 172.21.15.66:0/1965504515 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fcdc0005040 con 0x7fcdc40f51a0 2023-11-17T05:33:44.794 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:44.791+0000 7fcdc9d3f700 1 --2- 172.21.15.66:0/1965504515 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fcdc40f51a0 0x7fcdc40fd120 secure :-1 s=READY pgs=386 cs=0 l=1 rev1=1 rx=0x7fcdc0013510 tx=0x7fcdc000b040).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:33:44.794 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:44.791+0000 7fcdb67fc700 1 -- 172.21.15.66:0/1965504515 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7fcdc0009320 con 0x7fcdc40f51a0 2023-11-17T05:33:44.794 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:44.791+0000 7fcdcad41700 1 -- 172.21.15.66:0/1965504515 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fcdc40fb740 con 0x7fcdc40f51a0 2023-11-17T05:33:44.794 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:44.791+0000 7fcdb67fc700 1 -- 172.21.15.66:0/1965504515 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(25 keys) v1 ==== 1074+0+0 (secure 0 0 0) 0x7fcdc0013f00 con 0x7fcdc40f51a0 2023-11-17T05:33:44.795 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:44.791+0000 7fcdb67fc700 1 -- 172.21.15.66:0/1965504515 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7fcdc0014dc0 con 0x7fcdc40f51a0 2023-11-17T05:33:44.795 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:44.791+0000 7fcdcad41700 1 -- 172.21.15.66:0/1965504515 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fcdc40fbbd0 con 0x7fcdc40f51a0 2023-11-17T05:33:44.795 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:44.792+0000 7fcdb67fc700 1 -- 172.21.15.66:0/1965504515 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 17) v1 ==== 36269+0+0 (secure 0 0 0) 0x7fcdc00095f0 con 0x7fcdc40f51a0 2023-11-17T05:33:44.795 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:44.792+0000 7fcdcad41700 1 -- 172.21.15.66:0/1965504515 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fcdc4059100 con 0x7fcdc40f51a0 2023-11-17T05:33:44.795 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:44.792+0000 7fcdb67fc700 1 --2- 172.21.15.66:0/1965504515 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fcdb002ff60 0x7fcdb0032420 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:33:44.795 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:44.792+0000 7fcdb67fc700 1 -- 172.21.15.66:0/1965504515 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(5..5 src has 1..5) v4 ==== 1270+0+0 (secure 0 0 0) 0x7fcdc0041020 con 0x7fcdc40f51a0 2023-11-17T05:33:44.795 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:44.793+0000 7fcdc953e700 1 --2- 172.21.15.66:0/1965504515 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fcdb002ff60 0x7fcdb0032420 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:33:44.795 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:44.793+0000 7fcdc953e700 1 --2- 172.21.15.66:0/1965504515 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fcdb002ff60 0x7fcdb0032420 secure :-1 s=READY pgs=158 cs=0 l=1 rev1=1 rx=0x7fcdb80096a0 tx=0x7fcdb8006b40).ready entity=mgr.14164 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:33:44.799 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:44.797+0000 7fcdb67fc700 1 -- 172.21.15.66:0/1965504515 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+152212 (secure 0 0 0) 0x7fcdc0014420 con 0x7fcdc40f51a0 2023-11-17T05:33:45.015 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:45.014+0000 7fcdcad41700 1 -- 172.21.15.66:0/1965504515 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "mon dump", "format": "json"} v 0) v1 -- 0x7fcdc4045ad0 con 0x7fcdc40f51a0 2023-11-17T05:33:45.016 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:45.014+0000 7fcdb67fc700 1 -- 172.21.15.66:0/1965504515 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mon_command_ack([{"prefix": "mon dump", "format": "json"}]=0 dumped monmap epoch 1 v1) v1 ==== 95+0+690 (secure 0 0 0) 0x7fcdc001c920 con 0x7fcdc40f51a0 2023-11-17T05:33:45.016 INFO:teuthology.orchestra.run.smithi066.stdout: 2023-11-17T05:33:45.016 INFO:teuthology.orchestra.run.smithi066.stdout:{"epoch":1,"fsid":"ed6470e2-8508-11ee-8dbe-212e2dc638e7","modified":"2023-11-17T05:20:13.934038Z","created":"2023-11-17T05:20:13.934038Z","min_mon_release":16,"min_mon_release_name":"pacific","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging"],"optional":[]},"mons":[{"rank":0,"name":"smithi040","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:3300","nonce":0},{"type":"v1","addr":"172.21.15.40:6789","nonce":0}]},"addr":"172.21.15.40:6789/0","public_addr":"172.21.15.40:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2023-11-17T05:33:45.017 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:45.016+0000 7fcdcad41700 1 -- 172.21.15.66:0/1965504515 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fcdb002ff60 msgr2=0x7fcdb0032420 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:33:45.017 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:45.016+0000 7fcdcad41700 1 --2- 172.21.15.66:0/1965504515 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fcdb002ff60 0x7fcdb0032420 secure :-1 s=READY pgs=158 cs=0 l=1 rev1=1 rx=0x7fcdb80096a0 tx=0x7fcdb8006b40).stop 2023-11-17T05:33:45.018 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:45.016+0000 7fcdcad41700 1 -- 172.21.15.66:0/1965504515 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fcdc40f51a0 msgr2=0x7fcdc40fd120 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:33:45.018 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:45.016+0000 7fcdcad41700 1 --2- 172.21.15.66:0/1965504515 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fcdc40f51a0 0x7fcdc40fd120 secure :-1 s=READY pgs=386 cs=0 l=1 rev1=1 rx=0x7fcdc0013510 tx=0x7fcdc000b040).stop 2023-11-17T05:33:45.018 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:45.016+0000 7fcdcad41700 1 -- 172.21.15.66:0/1965504515 shutdown_connections 2023-11-17T05:33:45.018 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:45.016+0000 7fcdcad41700 1 --2- 172.21.15.66:0/1965504515 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fcdb002ff60 0x7fcdb0032420 unknown :-1 s=CLOSED pgs=158 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:33:45.018 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:45.016+0000 7fcdcad41700 1 --2- 172.21.15.66:0/1965504515 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fcdc40f51a0 0x7fcdc40fd120 unknown :-1 s=CLOSED pgs=386 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:33:45.018 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:45.016+0000 7fcdcad41700 1 -- 172.21.15.66:0/1965504515 >> 172.21.15.66:0/1965504515 conn(0x7fcdc40f09a0 msgr2=0x7fcdc405fe20 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:33:45.018 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:45.016+0000 7fcdcad41700 1 -- 172.21.15.66:0/1965504515 shutdown_connections 2023-11-17T05:33:45.018 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:45.016+0000 7fcdcad41700 1 -- 172.21.15.66:0/1965504515 wait complete. 2023-11-17T05:33:45.019 INFO:teuthology.orchestra.run.smithi066.stderr:dumped monmap epoch 1 2023-11-17T05:33:45.989 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:33:45 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:33:45.016077+0000 mon.smithi040 (mon.0) 307 : audit [DBG] from='client.? 172.21.15.66:0/1965504515' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-11-17T05:33:46.404 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2023-11-17T05:33:46.405 DEBUG:teuthology.orchestra.run.smithi066:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.4 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ed6470e2-8508-11ee-8dbe-212e2dc638e7 -- ceph mon dump -f json 2023-11-17T05:33:46.989 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:33:46 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:33:45.538375+0000 mgr.smithi040.ogcprl (mgr.14164) 388 : cluster [DBG] pgmap v372: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:33:47.610 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:47.608+0000 7f2889511700 1 -- 172.21.15.66:0/2604154428 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f28840f51a0 msgr2=0x7f28840f55c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:33:47.610 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:47.608+0000 7f2889511700 1 --2- 172.21.15.66:0/2604154428 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f28840f51a0 0x7f28840f55c0 secure :-1 s=READY pgs=387 cs=0 l=1 rev1=1 rx=0x7f2874009fb0 tx=0x7f2874009b30).stop 2023-11-17T05:33:47.611 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:47.608+0000 7f2889511700 1 -- 172.21.15.66:0/2604154428 shutdown_connections 2023-11-17T05:33:47.611 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:47.608+0000 7f2889511700 1 --2- 172.21.15.66:0/2604154428 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f28840f51a0 0x7f28840f55c0 unknown :-1 s=CLOSED pgs=387 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:33:47.611 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:47.608+0000 7f2889511700 1 -- 172.21.15.66:0/2604154428 >> 172.21.15.66:0/2604154428 conn(0x7f28840f09a0 msgr2=0x7f28840f2dc0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:33:47.611 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:47.608+0000 7f2889511700 1 -- 172.21.15.66:0/2604154428 shutdown_connections 2023-11-17T05:33:47.611 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:47.608+0000 7f2889511700 1 -- 172.21.15.66:0/2604154428 wait complete. 2023-11-17T05:33:47.611 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:47.608+0000 7f2889511700 1 Processor -- start 2023-11-17T05:33:47.611 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:47.609+0000 7f2889511700 1 -- start start 2023-11-17T05:33:47.611 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:47.609+0000 7f2889511700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f28840f51a0 0x7f28841086c0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:33:47.611 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:47.609+0000 7f2889511700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f2884108bb0 con 0x7f28840f51a0 2023-11-17T05:33:47.611 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:47.609+0000 7f2883fff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f28840f51a0 0x7f28841086c0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:33:47.611 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:47.609+0000 7f2883fff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f28840f51a0 0x7f28841086c0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.66:58692/0 (socket says 172.21.15.66:58692) 2023-11-17T05:33:47.611 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:47.609+0000 7f2883fff700 1 -- 172.21.15.66:0/746779913 learned_addr learned my addr 172.21.15.66:0/746779913 (peer_addr_for_me v2:172.21.15.66:0/0) 2023-11-17T05:33:47.612 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:47.609+0000 7f2883fff700 1 -- 172.21.15.66:0/746779913 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f2874005040 con 0x7f28840f51a0 2023-11-17T05:33:47.612 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:47.609+0000 7f2883fff700 1 --2- 172.21.15.66:0/746779913 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f28840f51a0 0x7f28841086c0 secure :-1 s=READY pgs=388 cs=0 l=1 rev1=1 rx=0x7f2874013510 tx=0x7f287400b040).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:33:47.613 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:47.609+0000 7f2880ff9700 1 -- 172.21.15.66:0/746779913 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f2874009320 con 0x7f28840f51a0 2023-11-17T05:33:47.613 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:47.610+0000 7f2889511700 1 -- 172.21.15.66:0/746779913 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f2884108db0 con 0x7f28840f51a0 2023-11-17T05:33:47.613 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:47.610+0000 7f2880ff9700 1 -- 172.21.15.66:0/746779913 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(25 keys) v1 ==== 1074+0+0 (secure 0 0 0) 0x7f2874013f00 con 0x7f28840f51a0 2023-11-17T05:33:47.613 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:47.610+0000 7f2880ff9700 1 -- 172.21.15.66:0/746779913 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f2874014dc0 con 0x7f28840f51a0 2023-11-17T05:33:47.613 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:47.610+0000 7f2889511700 1 -- 172.21.15.66:0/746779913 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f2884109240 con 0x7f28840f51a0 2023-11-17T05:33:47.613 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:47.610+0000 7f2880ff9700 1 -- 172.21.15.66:0/746779913 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 17) v1 ==== 36269+0+0 (secure 0 0 0) 0x7f28740095f0 con 0x7f28840f51a0 2023-11-17T05:33:47.614 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:47.610+0000 7f2889511700 1 -- 172.21.15.66:0/746779913 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f2884045ad0 con 0x7f28840f51a0 2023-11-17T05:33:47.614 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:47.611+0000 7f2880ff9700 1 --2- 172.21.15.66:0/746779913 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f287002ffb0 0x7f2870032470 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:33:47.614 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:47.611+0000 7f2880ff9700 1 -- 172.21.15.66:0/746779913 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(5..5 src has 1..5) v4 ==== 1270+0+0 (secure 0 0 0) 0x7f2874041100 con 0x7f28840f51a0 2023-11-17T05:33:47.614 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:47.611+0000 7f28837fe700 1 --2- 172.21.15.66:0/746779913 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f287002ffb0 0x7f2870032470 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:33:47.614 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:47.611+0000 7f28837fe700 1 --2- 172.21.15.66:0/746779913 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f287002ffb0 0x7f2870032470 secure :-1 s=READY pgs=159 cs=0 l=1 rev1=1 rx=0x7f286c0096a0 tx=0x7f286c006b40).ready entity=mgr.14164 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:33:47.616 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:47.615+0000 7f2880ff9700 1 -- 172.21.15.66:0/746779913 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+152212 (secure 0 0 0) 0x7f2874014420 con 0x7f28840f51a0 2023-11-17T05:33:47.834 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:47.832+0000 7f2889511700 1 -- 172.21.15.66:0/746779913 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "mon dump", "format": "json"} v 0) v1 -- 0x7f288410a0f0 con 0x7f28840f51a0 2023-11-17T05:33:47.834 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:47.833+0000 7f2880ff9700 1 -- 172.21.15.66:0/746779913 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mon_command_ack([{"prefix": "mon dump", "format": "json"}]=0 dumped monmap epoch 1 v1) v1 ==== 95+0+690 (secure 0 0 0) 0x7f2874010230 con 0x7f28840f51a0 2023-11-17T05:33:47.834 INFO:teuthology.orchestra.run.smithi066.stdout: 2023-11-17T05:33:47.835 INFO:teuthology.orchestra.run.smithi066.stdout:{"epoch":1,"fsid":"ed6470e2-8508-11ee-8dbe-212e2dc638e7","modified":"2023-11-17T05:20:13.934038Z","created":"2023-11-17T05:20:13.934038Z","min_mon_release":16,"min_mon_release_name":"pacific","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging"],"optional":[]},"mons":[{"rank":0,"name":"smithi040","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:3300","nonce":0},{"type":"v1","addr":"172.21.15.40:6789","nonce":0}]},"addr":"172.21.15.40:6789/0","public_addr":"172.21.15.40:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2023-11-17T05:33:47.836 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:47.835+0000 7f2889511700 1 -- 172.21.15.66:0/746779913 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f287002ffb0 msgr2=0x7f2870032470 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:33:47.836 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:47.835+0000 7f2889511700 1 --2- 172.21.15.66:0/746779913 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f287002ffb0 0x7f2870032470 secure :-1 s=READY pgs=159 cs=0 l=1 rev1=1 rx=0x7f286c0096a0 tx=0x7f286c006b40).stop 2023-11-17T05:33:47.836 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:47.835+0000 7f2889511700 1 -- 172.21.15.66:0/746779913 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f28840f51a0 msgr2=0x7f28841086c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:33:47.836 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:47.835+0000 7f2889511700 1 --2- 172.21.15.66:0/746779913 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f28840f51a0 0x7f28841086c0 secure :-1 s=READY pgs=388 cs=0 l=1 rev1=1 rx=0x7f2874013510 tx=0x7f287400b040).stop 2023-11-17T05:33:47.836 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:47.835+0000 7f2889511700 1 -- 172.21.15.66:0/746779913 shutdown_connections 2023-11-17T05:33:47.837 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:47.835+0000 7f2889511700 1 --2- 172.21.15.66:0/746779913 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f287002ffb0 0x7f2870032470 unknown :-1 s=CLOSED pgs=159 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:33:47.837 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:47.835+0000 7f2889511700 1 --2- 172.21.15.66:0/746779913 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f28840f51a0 0x7f28841086c0 unknown :-1 s=CLOSED pgs=388 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:33:47.837 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:47.835+0000 7f2889511700 1 -- 172.21.15.66:0/746779913 >> 172.21.15.66:0/746779913 conn(0x7f28840f09a0 msgr2=0x7f288405fde0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:33:47.837 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:47.835+0000 7f2889511700 1 -- 172.21.15.66:0/746779913 shutdown_connections 2023-11-17T05:33:47.837 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:47.835+0000 7f2889511700 1 -- 172.21.15.66:0/746779913 wait complete. 2023-11-17T05:33:47.837 INFO:teuthology.orchestra.run.smithi066.stderr:dumped monmap epoch 1 2023-11-17T05:33:47.989 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:33:47 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:33:47.538807+0000 mgr.smithi040.ogcprl (mgr.14164) 389 : cluster [DBG] pgmap v373: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:33:48.989 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:33:48 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:33:47.834939+0000 mon.smithi040 (mon.0) 308 : audit [DBG] from='client.? 172.21.15.66:0/746779913' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-11-17T05:33:49.471 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2023-11-17T05:33:49.472 DEBUG:teuthology.orchestra.run.smithi066:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.4 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ed6470e2-8508-11ee-8dbe-212e2dc638e7 -- ceph mon dump -f json 2023-11-17T05:33:50.688 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:50.676+0000 7fa55ae42700 1 -- 172.21.15.66:0/1412067024 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fa5540f5410 msgr2=0x7fa5540f5830 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:33:50.689 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:50.676+0000 7fa55ae42700 1 --2- 172.21.15.66:0/1412067024 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fa5540f5410 0x7fa5540f5830 secure :-1 s=READY pgs=389 cs=0 l=1 rev1=1 rx=0x7fa548009fb0 tx=0x7fa548009b30).stop 2023-11-17T05:33:50.689 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:50.676+0000 7fa55ae42700 1 -- 172.21.15.66:0/1412067024 shutdown_connections 2023-11-17T05:33:50.689 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:50.676+0000 7fa55ae42700 1 --2- 172.21.15.66:0/1412067024 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fa5540f5410 0x7fa5540f5830 unknown :-1 s=CLOSED pgs=389 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:33:50.689 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:50.676+0000 7fa55ae42700 1 -- 172.21.15.66:0/1412067024 >> 172.21.15.66:0/1412067024 conn(0x7fa5540f0b70 msgr2=0x7fa5540f2f90 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:33:50.689 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:50.676+0000 7fa55ae42700 1 -- 172.21.15.66:0/1412067024 shutdown_connections 2023-11-17T05:33:50.689 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:50.676+0000 7fa55ae42700 1 -- 172.21.15.66:0/1412067024 wait complete. 2023-11-17T05:33:50.689 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:50.677+0000 7fa55ae42700 1 Processor -- start 2023-11-17T05:33:50.689 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:50.677+0000 7fa55ae42700 1 -- start start 2023-11-17T05:33:50.689 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:50.677+0000 7fa55ae42700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fa5540f5410 0x7fa554106720 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:33:50.689 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:50.677+0000 7fa55ae42700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fa554106c10 con 0x7fa5540f5410 2023-11-17T05:33:50.690 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:50.677+0000 7fa558bde700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fa5540f5410 0x7fa554106720 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:33:50.690 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:50.677+0000 7fa558bde700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fa5540f5410 0x7fa554106720 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.66:58710/0 (socket says 172.21.15.66:58710) 2023-11-17T05:33:50.690 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:50.677+0000 7fa558bde700 1 -- 172.21.15.66:0/111727588 learned_addr learned my addr 172.21.15.66:0/111727588 (peer_addr_for_me v2:172.21.15.66:0/0) 2023-11-17T05:33:50.690 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:50.678+0000 7fa558bde700 1 -- 172.21.15.66:0/111727588 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fa548005040 con 0x7fa5540f5410 2023-11-17T05:33:50.690 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:50.678+0000 7fa558bde700 1 --2- 172.21.15.66:0/111727588 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fa5540f5410 0x7fa554106720 secure :-1 s=READY pgs=390 cs=0 l=1 rev1=1 rx=0x7fa548013510 tx=0x7fa54800b040).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:33:50.690 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:50.678+0000 7fa5517fa700 1 -- 172.21.15.66:0/111727588 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7fa548009320 con 0x7fa5540f5410 2023-11-17T05:33:50.690 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:50.678+0000 7fa5517fa700 1 -- 172.21.15.66:0/111727588 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(25 keys) v1 ==== 1074+0+0 (secure 0 0 0) 0x7fa548013f00 con 0x7fa5540f5410 2023-11-17T05:33:50.691 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:50.678+0000 7fa55ae42700 1 -- 172.21.15.66:0/111727588 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fa554106e10 con 0x7fa5540f5410 2023-11-17T05:33:50.691 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:50.678+0000 7fa5517fa700 1 -- 172.21.15.66:0/111727588 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7fa548014d90 con 0x7fa5540f5410 2023-11-17T05:33:50.691 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:50.678+0000 7fa55ae42700 1 -- 172.21.15.66:0/111727588 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fa5541072a0 con 0x7fa5540f5410 2023-11-17T05:33:50.691 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:50.679+0000 7fa5517fa700 1 -- 172.21.15.66:0/111727588 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 17) v1 ==== 36269+0+0 (secure 0 0 0) 0x7fa5480263a0 con 0x7fa5540f5410 2023-11-17T05:33:50.691 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:50.679+0000 7fa5517fa700 1 --2- 172.21.15.66:0/111727588 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fa54402ff60 0x7fa544032420 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:33:50.691 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:50.679+0000 7fa5517fa700 1 -- 172.21.15.66:0/111727588 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(5..5 src has 1..5) v4 ==== 1270+0+0 (secure 0 0 0) 0x7fa548041110 con 0x7fa5540f5410 2023-11-17T05:33:50.691 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:50.679+0000 7fa55ae42700 1 -- 172.21.15.66:0/111727588 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fa554045ad0 con 0x7fa5540f5410 2023-11-17T05:33:50.691 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:50.679+0000 7fa553fff700 1 --2- 172.21.15.66:0/111727588 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fa54402ff60 0x7fa544032420 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:33:50.691 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:50.680+0000 7fa553fff700 1 --2- 172.21.15.66:0/111727588 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fa54402ff60 0x7fa544032420 secure :-1 s=READY pgs=160 cs=0 l=1 rev1=1 rx=0x7fa5400096a0 tx=0x7fa540006b40).ready entity=mgr.14164 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:33:50.691 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:50.684+0000 7fa5517fa700 1 -- 172.21.15.66:0/111727588 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+152212 (secure 0 0 0) 0x7fa548012920 con 0x7fa5540f5410 2023-11-17T05:33:50.906 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:50.904+0000 7fa55ae42700 1 -- 172.21.15.66:0/111727588 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "mon dump", "format": "json"} v 0) v1 -- 0x7fa554107fc0 con 0x7fa5540f5410 2023-11-17T05:33:50.906 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:50.906+0000 7fa5517fa700 1 -- 172.21.15.66:0/111727588 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mon_command_ack([{"prefix": "mon dump", "format": "json"}]=0 dumped monmap epoch 1 v1) v1 ==== 95+0+690 (secure 0 0 0) 0x7fa54802b020 con 0x7fa5540f5410 2023-11-17T05:33:50.907 INFO:teuthology.orchestra.run.smithi066.stdout: 2023-11-17T05:33:50.907 INFO:teuthology.orchestra.run.smithi066.stdout:{"epoch":1,"fsid":"ed6470e2-8508-11ee-8dbe-212e2dc638e7","modified":"2023-11-17T05:20:13.934038Z","created":"2023-11-17T05:20:13.934038Z","min_mon_release":16,"min_mon_release_name":"pacific","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging"],"optional":[]},"mons":[{"rank":0,"name":"smithi040","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:3300","nonce":0},{"type":"v1","addr":"172.21.15.40:6789","nonce":0}]},"addr":"172.21.15.40:6789/0","public_addr":"172.21.15.40:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2023-11-17T05:33:50.908 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:50.907+0000 7fa55ae42700 1 -- 172.21.15.66:0/111727588 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fa54402ff60 msgr2=0x7fa544032420 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:33:50.908 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:50.907+0000 7fa55ae42700 1 --2- 172.21.15.66:0/111727588 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fa54402ff60 0x7fa544032420 secure :-1 s=READY pgs=160 cs=0 l=1 rev1=1 rx=0x7fa5400096a0 tx=0x7fa540006b40).stop 2023-11-17T05:33:50.909 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:50.907+0000 7fa55ae42700 1 -- 172.21.15.66:0/111727588 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fa5540f5410 msgr2=0x7fa554106720 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:33:50.909 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:50.907+0000 7fa55ae42700 1 --2- 172.21.15.66:0/111727588 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fa5540f5410 0x7fa554106720 secure :-1 s=READY pgs=390 cs=0 l=1 rev1=1 rx=0x7fa548013510 tx=0x7fa54800b040).stop 2023-11-17T05:33:50.909 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:50.907+0000 7fa55ae42700 1 -- 172.21.15.66:0/111727588 shutdown_connections 2023-11-17T05:33:50.909 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:50.907+0000 7fa55ae42700 1 --2- 172.21.15.66:0/111727588 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fa54402ff60 0x7fa544032420 unknown :-1 s=CLOSED pgs=160 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:33:50.909 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:50.907+0000 7fa55ae42700 1 --2- 172.21.15.66:0/111727588 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fa5540f5410 0x7fa554106720 unknown :-1 s=CLOSED pgs=390 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:33:50.909 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:50.907+0000 7fa55ae42700 1 -- 172.21.15.66:0/111727588 >> 172.21.15.66:0/111727588 conn(0x7fa5540f0b70 msgr2=0x7fa55405fe60 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:33:50.910 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:50.907+0000 7fa55ae42700 1 -- 172.21.15.66:0/111727588 shutdown_connections 2023-11-17T05:33:50.910 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:50.907+0000 7fa55ae42700 1 -- 172.21.15.66:0/111727588 wait complete. 2023-11-17T05:33:50.910 INFO:teuthology.orchestra.run.smithi066.stderr:dumped monmap epoch 1 2023-11-17T05:33:50.988 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:33:50 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:33:49.539199+0000 mgr.smithi040.ogcprl (mgr.14164) 390 : cluster [DBG] pgmap v374: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:33:51.988 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:33:51 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:33:50.906505+0000 mon.smithi040 (mon.0) 309 : audit [DBG] from='client.? 172.21.15.66:0/111727588' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-11-17T05:33:52.298 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2023-11-17T05:33:52.298 DEBUG:teuthology.orchestra.run.smithi066:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.4 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ed6470e2-8508-11ee-8dbe-212e2dc638e7 -- ceph mon dump -f json 2023-11-17T05:33:52.989 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:33:52 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:33:51.539491+0000 mgr.smithi040.ogcprl (mgr.14164) 391 : cluster [DBG] pgmap v375: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:33:53.505 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:53.501+0000 7faf81076700 1 -- 172.21.15.66:0/3283079224 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7faf7c0f6510 msgr2=0x7faf7c0f6930 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:33:53.505 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:53.501+0000 7faf81076700 1 --2- 172.21.15.66:0/3283079224 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7faf7c0f6510 0x7faf7c0f6930 secure :-1 s=READY pgs=391 cs=0 l=1 rev1=1 rx=0x7faf6c009fb0 tx=0x7faf6c009b30).stop 2023-11-17T05:33:53.505 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:53.501+0000 7faf81076700 1 -- 172.21.15.66:0/3283079224 shutdown_connections 2023-11-17T05:33:53.505 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:53.501+0000 7faf81076700 1 --2- 172.21.15.66:0/3283079224 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7faf7c0f6510 0x7faf7c0f6930 unknown :-1 s=CLOSED pgs=391 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:33:53.505 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:53.501+0000 7faf81076700 1 -- 172.21.15.66:0/3283079224 >> 172.21.15.66:0/3283079224 conn(0x7faf7c0f1c70 msgr2=0x7faf7c0f4090 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:33:53.506 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:53.501+0000 7faf81076700 1 -- 172.21.15.66:0/3283079224 shutdown_connections 2023-11-17T05:33:53.506 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:53.501+0000 7faf81076700 1 -- 172.21.15.66:0/3283079224 wait complete. 2023-11-17T05:33:53.506 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:53.502+0000 7faf81076700 1 Processor -- start 2023-11-17T05:33:53.506 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:53.502+0000 7faf81076700 1 -- start start 2023-11-17T05:33:53.506 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:53.502+0000 7faf81076700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7faf7c0f6510 0x7faf7c106730 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:33:53.506 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:53.502+0000 7faf81076700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7faf7c106c20 con 0x7faf7c0f6510 2023-11-17T05:33:53.507 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:53.502+0000 7faf7ad9d700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7faf7c0f6510 0x7faf7c106730 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:33:53.507 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:53.502+0000 7faf7ad9d700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7faf7c0f6510 0x7faf7c106730 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.66:46430/0 (socket says 172.21.15.66:46430) 2023-11-17T05:33:53.507 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:53.502+0000 7faf7ad9d700 1 -- 172.21.15.66:0/923212016 learned_addr learned my addr 172.21.15.66:0/923212016 (peer_addr_for_me v2:172.21.15.66:0/0) 2023-11-17T05:33:53.507 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:53.503+0000 7faf7ad9d700 1 -- 172.21.15.66:0/923212016 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7faf6c005040 con 0x7faf7c0f6510 2023-11-17T05:33:53.507 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:53.503+0000 7faf7ad9d700 1 --2- 172.21.15.66:0/923212016 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7faf7c0f6510 0x7faf7c106730 secure :-1 s=READY pgs=392 cs=0 l=1 rev1=1 rx=0x7faf6c013510 tx=0x7faf6c00b040).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:33:53.508 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:53.503+0000 7faf737fe700 1 -- 172.21.15.66:0/923212016 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7faf6c009320 con 0x7faf7c0f6510 2023-11-17T05:33:53.508 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:53.503+0000 7faf737fe700 1 -- 172.21.15.66:0/923212016 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(25 keys) v1 ==== 1074+0+0 (secure 0 0 0) 0x7faf6c013f00 con 0x7faf7c0f6510 2023-11-17T05:33:53.508 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:53.503+0000 7faf81076700 1 -- 172.21.15.66:0/923212016 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7faf7c106e20 con 0x7faf7c0f6510 2023-11-17T05:33:53.508 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:53.503+0000 7faf737fe700 1 -- 172.21.15.66:0/923212016 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7faf6c014d90 con 0x7faf7c0f6510 2023-11-17T05:33:53.508 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:53.503+0000 7faf81076700 1 -- 172.21.15.66:0/923212016 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7faf7c1072b0 con 0x7faf7c0f6510 2023-11-17T05:33:53.510 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:53.504+0000 7faf737fe700 1 -- 172.21.15.66:0/923212016 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 17) v1 ==== 36269+0+0 (secure 0 0 0) 0x7faf6c0095f0 con 0x7faf7c0f6510 2023-11-17T05:33:53.510 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:53.504+0000 7faf81076700 1 -- 172.21.15.66:0/923212016 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7faf7c045ad0 con 0x7faf7c0f6510 2023-11-17T05:33:53.510 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:53.504+0000 7faf737fe700 1 --2- 172.21.15.66:0/923212016 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7faf6802ff60 0x7faf68032420 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:33:53.510 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:53.504+0000 7faf737fe700 1 -- 172.21.15.66:0/923212016 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(5..5 src has 1..5) v4 ==== 1270+0+0 (secure 0 0 0) 0x7faf6c041000 con 0x7faf7c0f6510 2023-11-17T05:33:53.510 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:53.504+0000 7faf7a59c700 1 --2- 172.21.15.66:0/923212016 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7faf6802ff60 0x7faf68032420 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:33:53.511 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:53.505+0000 7faf7a59c700 1 --2- 172.21.15.66:0/923212016 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7faf6802ff60 0x7faf68032420 secure :-1 s=READY pgs=161 cs=0 l=1 rev1=1 rx=0x7faf640096a0 tx=0x7faf64006b40).ready entity=mgr.14164 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:33:53.511 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:53.509+0000 7faf737fe700 1 -- 172.21.15.66:0/923212016 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+152212 (secure 0 0 0) 0x7faf6c014420 con 0x7faf7c0f6510 2023-11-17T05:33:53.730 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:53.729+0000 7faf81076700 1 -- 172.21.15.66:0/923212016 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "mon dump", "format": "json"} v 0) v1 -- 0x7faf7c0fbf60 con 0x7faf7c0f6510 2023-11-17T05:33:53.731 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:53.730+0000 7faf737fe700 1 -- 172.21.15.66:0/923212016 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mon_command_ack([{"prefix": "mon dump", "format": "json"}]=0 dumped monmap epoch 1 v1) v1 ==== 95+0+690 (secure 0 0 0) 0x7faf6c01d7f0 con 0x7faf7c0f6510 2023-11-17T05:33:53.731 INFO:teuthology.orchestra.run.smithi066.stdout: 2023-11-17T05:33:53.731 INFO:teuthology.orchestra.run.smithi066.stdout:{"epoch":1,"fsid":"ed6470e2-8508-11ee-8dbe-212e2dc638e7","modified":"2023-11-17T05:20:13.934038Z","created":"2023-11-17T05:20:13.934038Z","min_mon_release":16,"min_mon_release_name":"pacific","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging"],"optional":[]},"mons":[{"rank":0,"name":"smithi040","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:3300","nonce":0},{"type":"v1","addr":"172.21.15.40:6789","nonce":0}]},"addr":"172.21.15.40:6789/0","public_addr":"172.21.15.40:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2023-11-17T05:33:53.733 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:53.731+0000 7faf81076700 1 -- 172.21.15.66:0/923212016 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7faf6802ff60 msgr2=0x7faf68032420 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:33:53.733 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:53.731+0000 7faf81076700 1 --2- 172.21.15.66:0/923212016 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7faf6802ff60 0x7faf68032420 secure :-1 s=READY pgs=161 cs=0 l=1 rev1=1 rx=0x7faf640096a0 tx=0x7faf64006b40).stop 2023-11-17T05:33:53.733 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:53.731+0000 7faf81076700 1 -- 172.21.15.66:0/923212016 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7faf7c0f6510 msgr2=0x7faf7c106730 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:33:53.733 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:53.731+0000 7faf81076700 1 --2- 172.21.15.66:0/923212016 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7faf7c0f6510 0x7faf7c106730 secure :-1 s=READY pgs=392 cs=0 l=1 rev1=1 rx=0x7faf6c013510 tx=0x7faf6c00b040).stop 2023-11-17T05:33:53.733 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:53.731+0000 7faf81076700 1 -- 172.21.15.66:0/923212016 shutdown_connections 2023-11-17T05:33:53.733 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:53.731+0000 7faf81076700 1 --2- 172.21.15.66:0/923212016 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7faf6802ff60 0x7faf68032420 unknown :-1 s=CLOSED pgs=161 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:33:53.733 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:53.731+0000 7faf81076700 1 --2- 172.21.15.66:0/923212016 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7faf7c0f6510 0x7faf7c106730 unknown :-1 s=CLOSED pgs=392 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:33:53.733 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:53.731+0000 7faf81076700 1 -- 172.21.15.66:0/923212016 >> 172.21.15.66:0/923212016 conn(0x7faf7c0f1c70 msgr2=0x7faf7c0f2760 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:33:53.733 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:53.731+0000 7faf81076700 1 -- 172.21.15.66:0/923212016 shutdown_connections 2023-11-17T05:33:53.733 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:53.731+0000 7faf81076700 1 -- 172.21.15.66:0/923212016 wait complete. 2023-11-17T05:33:53.734 INFO:teuthology.orchestra.run.smithi066.stderr:dumped monmap epoch 1 2023-11-17T05:33:53.988 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:33:53 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:33:53.539783+0000 mgr.smithi040.ogcprl (mgr.14164) 392 : cluster [DBG] pgmap v376: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:33:54.988 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:33:54 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:33:53.731485+0000 mon.smithi040 (mon.0) 310 : audit [DBG] from='client.? 172.21.15.66:0/923212016' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-11-17T05:33:55.148 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2023-11-17T05:33:55.149 DEBUG:teuthology.orchestra.run.smithi066:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.4 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ed6470e2-8508-11ee-8dbe-212e2dc638e7 -- ceph mon dump -f json 2023-11-17T05:33:55.989 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:33:55 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:33:55.540132+0000 mgr.smithi040.ogcprl (mgr.14164) 393 : cluster [DBG] pgmap v377: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:33:56.294 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:56.293+0000 7f47af59e700 1 -- 172.21.15.66:0/804230206 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f47b00601e0 msgr2=0x7f47b00f8f20 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:33:56.294 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:56.293+0000 7f47af59e700 1 --2- 172.21.15.66:0/804230206 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f47b00601e0 0x7f47b00f8f20 secure :-1 s=READY pgs=393 cs=0 l=1 rev1=1 rx=0x7f47a0009fb0 tx=0x7f47a0009b30).stop 2023-11-17T05:33:56.294 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:56.293+0000 7f47af59e700 1 -- 172.21.15.66:0/804230206 shutdown_connections 2023-11-17T05:33:56.295 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:56.293+0000 7f47af59e700 1 --2- 172.21.15.66:0/804230206 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f47b00601e0 0x7f47b00f8f20 unknown :-1 s=CLOSED pgs=393 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:33:56.295 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:56.293+0000 7f47af59e700 1 -- 172.21.15.66:0/804230206 >> 172.21.15.66:0/804230206 conn(0x7f47b00f09a0 msgr2=0x7f47b00f2dc0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:33:56.295 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:56.293+0000 7f47af59e700 1 -- 172.21.15.66:0/804230206 shutdown_connections 2023-11-17T05:33:56.295 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:56.293+0000 7f47af59e700 1 -- 172.21.15.66:0/804230206 wait complete. 2023-11-17T05:33:56.297 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:56.293+0000 7f47af59e700 1 Processor -- start 2023-11-17T05:33:56.297 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:56.294+0000 7f47af59e700 1 -- start start 2023-11-17T05:33:56.297 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:56.294+0000 7f47af59e700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f47b00601e0 0x7f47b00f69a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:33:56.297 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:56.294+0000 7f47af59e700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f47b00f4e20 con 0x7f47b00601e0 2023-11-17T05:33:56.297 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:56.294+0000 7f47ae59c700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f47b00601e0 0x7f47b00f69a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:33:56.297 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:56.294+0000 7f47ae59c700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f47b00601e0 0x7f47b00f69a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.66:46448/0 (socket says 172.21.15.66:46448) 2023-11-17T05:33:56.297 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:56.294+0000 7f47ae59c700 1 -- 172.21.15.66:0/2411967831 learned_addr learned my addr 172.21.15.66:0/2411967831 (peer_addr_for_me v2:172.21.15.66:0/0) 2023-11-17T05:33:56.297 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:56.294+0000 7f47ae59c700 1 -- 172.21.15.66:0/2411967831 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f47a0005040 con 0x7f47b00601e0 2023-11-17T05:33:56.297 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:56.294+0000 7f47ae59c700 1 --2- 172.21.15.66:0/2411967831 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f47b00601e0 0x7f47b00f69a0 secure :-1 s=READY pgs=394 cs=0 l=1 rev1=1 rx=0x7f47a0013510 tx=0x7f47a000b040).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:33:56.297 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:56.294+0000 7f479effd700 1 -- 172.21.15.66:0/2411967831 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f47a0009320 con 0x7f47b00601e0 2023-11-17T05:33:56.297 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:56.294+0000 7f47af59e700 1 -- 172.21.15.66:0/2411967831 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f47b00f4fc0 con 0x7f47b00601e0 2023-11-17T05:33:56.297 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:56.294+0000 7f479effd700 1 -- 172.21.15.66:0/2411967831 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(25 keys) v1 ==== 1074+0+0 (secure 0 0 0) 0x7f47a0013f00 con 0x7f47b00601e0 2023-11-17T05:33:56.298 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:56.294+0000 7f479effd700 1 -- 172.21.15.66:0/2411967831 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f47a0014d90 con 0x7f47b00601e0 2023-11-17T05:33:56.298 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:56.295+0000 7f47af59e700 1 -- 172.21.15.66:0/2411967831 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f47b00f5450 con 0x7f47b00601e0 2023-11-17T05:33:56.298 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:56.295+0000 7f479effd700 1 -- 172.21.15.66:0/2411967831 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 17) v1 ==== 36269+0+0 (secure 0 0 0) 0x7f47a00095f0 con 0x7f47b00601e0 2023-11-17T05:33:56.298 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:56.295+0000 7f47af59e700 1 -- 172.21.15.66:0/2411967831 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f47b0059100 con 0x7f47b00601e0 2023-11-17T05:33:56.298 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:56.295+0000 7f479effd700 1 --2- 172.21.15.66:0/2411967831 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f479802ff60 0x7f4798032420 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:33:56.298 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:56.295+0000 7f479effd700 1 -- 172.21.15.66:0/2411967831 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(5..5 src has 1..5) v4 ==== 1270+0+0 (secure 0 0 0) 0x7f47a0041000 con 0x7f47b00601e0 2023-11-17T05:33:56.298 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:56.296+0000 7f47add9b700 1 --2- 172.21.15.66:0/2411967831 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f479802ff60 0x7f4798032420 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:33:56.298 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:56.296+0000 7f47add9b700 1 --2- 172.21.15.66:0/2411967831 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f479802ff60 0x7f4798032420 secure :-1 s=READY pgs=162 cs=0 l=1 rev1=1 rx=0x7f47a40096a0 tx=0x7f47a4006b40).ready entity=mgr.14164 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:33:56.301 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:56.300+0000 7f479effd700 1 -- 172.21.15.66:0/2411967831 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+152212 (secure 0 0 0) 0x7f47a0014420 con 0x7f47b00601e0 2023-11-17T05:33:56.519 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:56.517+0000 7f47af59e700 1 -- 172.21.15.66:0/2411967831 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "mon dump", "format": "json"} v 0) v1 -- 0x7f47b00fbcd0 con 0x7f47b00601e0 2023-11-17T05:33:56.520 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:56.518+0000 7f479effd700 1 -- 172.21.15.66:0/2411967831 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mon_command_ack([{"prefix": "mon dump", "format": "json"}]=0 dumped monmap epoch 1 v1) v1 ==== 95+0+690 (secure 0 0 0) 0x7f47a0010230 con 0x7f47b00601e0 2023-11-17T05:33:56.520 INFO:teuthology.orchestra.run.smithi066.stdout: 2023-11-17T05:33:56.520 INFO:teuthology.orchestra.run.smithi066.stdout:{"epoch":1,"fsid":"ed6470e2-8508-11ee-8dbe-212e2dc638e7","modified":"2023-11-17T05:20:13.934038Z","created":"2023-11-17T05:20:13.934038Z","min_mon_release":16,"min_mon_release_name":"pacific","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging"],"optional":[]},"mons":[{"rank":0,"name":"smithi040","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:3300","nonce":0},{"type":"v1","addr":"172.21.15.40:6789","nonce":0}]},"addr":"172.21.15.40:6789/0","public_addr":"172.21.15.40:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2023-11-17T05:33:56.522 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:56.520+0000 7f47af59e700 1 -- 172.21.15.66:0/2411967831 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f479802ff60 msgr2=0x7f4798032420 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:33:56.522 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:56.520+0000 7f47af59e700 1 --2- 172.21.15.66:0/2411967831 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f479802ff60 0x7f4798032420 secure :-1 s=READY pgs=162 cs=0 l=1 rev1=1 rx=0x7f47a40096a0 tx=0x7f47a4006b40).stop 2023-11-17T05:33:56.522 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:56.520+0000 7f47af59e700 1 -- 172.21.15.66:0/2411967831 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f47b00601e0 msgr2=0x7f47b00f69a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:33:56.522 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:56.520+0000 7f47af59e700 1 --2- 172.21.15.66:0/2411967831 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f47b00601e0 0x7f47b00f69a0 secure :-1 s=READY pgs=394 cs=0 l=1 rev1=1 rx=0x7f47a0013510 tx=0x7f47a000b040).stop 2023-11-17T05:33:56.523 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:56.520+0000 7f47af59e700 1 -- 172.21.15.66:0/2411967831 shutdown_connections 2023-11-17T05:33:56.523 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:56.520+0000 7f47af59e700 1 --2- 172.21.15.66:0/2411967831 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f479802ff60 0x7f4798032420 unknown :-1 s=CLOSED pgs=162 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:33:56.523 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:56.520+0000 7f47af59e700 1 --2- 172.21.15.66:0/2411967831 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f47b00601e0 0x7f47b00f69a0 unknown :-1 s=CLOSED pgs=394 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:33:56.523 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:56.520+0000 7f47af59e700 1 -- 172.21.15.66:0/2411967831 >> 172.21.15.66:0/2411967831 conn(0x7f47b00f09a0 msgr2=0x7f47b00f1420 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:33:56.523 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:56.520+0000 7f47af59e700 1 -- 172.21.15.66:0/2411967831 shutdown_connections 2023-11-17T05:33:56.523 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:56.520+0000 7f47af59e700 1 -- 172.21.15.66:0/2411967831 wait complete. 2023-11-17T05:33:56.523 INFO:teuthology.orchestra.run.smithi066.stderr:dumped monmap epoch 1 2023-11-17T05:33:56.989 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:33:56 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:33:56.519892+0000 mon.smithi040 (mon.0) 311 : audit [DBG] from='client.? 172.21.15.66:0/2411967831' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-11-17T05:33:57.920 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2023-11-17T05:33:57.920 DEBUG:teuthology.orchestra.run.smithi066:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.4 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ed6470e2-8508-11ee-8dbe-212e2dc638e7 -- ceph mon dump -f json 2023-11-17T05:33:57.988 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:33:57 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:33:57.540371+0000 mgr.smithi040.ogcprl (mgr.14164) 394 : cluster [DBG] pgmap v378: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:33:59.008 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:59.006+0000 7fa94e8e6700 1 -- 172.21.15.66:0/157211839 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fa9480f47c0 msgr2=0x7fa9480f4be0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:33:59.008 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:59.006+0000 7fa94e8e6700 1 --2- 172.21.15.66:0/157211839 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fa9480f47c0 0x7fa9480f4be0 secure :-1 s=READY pgs=395 cs=0 l=1 rev1=1 rx=0x7fa938009fb0 tx=0x7fa938009b30).stop 2023-11-17T05:33:59.008 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:59.006+0000 7fa94e8e6700 1 -- 172.21.15.66:0/157211839 shutdown_connections 2023-11-17T05:33:59.008 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:59.006+0000 7fa94e8e6700 1 --2- 172.21.15.66:0/157211839 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fa9480f47c0 0x7fa9480f4be0 unknown :-1 s=CLOSED pgs=395 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:33:59.008 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:59.006+0000 7fa94e8e6700 1 -- 172.21.15.66:0/157211839 >> 172.21.15.66:0/157211839 conn(0x7fa9480f02e0 msgr2=0x7fa9480f2700 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:33:59.008 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:59.006+0000 7fa94e8e6700 1 -- 172.21.15.66:0/157211839 shutdown_connections 2023-11-17T05:33:59.008 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:59.006+0000 7fa94e8e6700 1 -- 172.21.15.66:0/157211839 wait complete. 2023-11-17T05:33:59.008 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:59.007+0000 7fa94e8e6700 1 Processor -- start 2023-11-17T05:33:59.011 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:59.007+0000 7fa94e8e6700 1 -- start start 2023-11-17T05:33:59.011 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:59.007+0000 7fa94e8e6700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fa9480f47c0 0x7fa948106700 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:33:59.011 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:59.007+0000 7fa94e8e6700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fa948106bf0 con 0x7fa9480f47c0 2023-11-17T05:33:59.011 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:59.007+0000 7fa947fff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fa9480f47c0 0x7fa948106700 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:33:59.011 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:59.007+0000 7fa947fff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fa9480f47c0 0x7fa948106700 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.66:46464/0 (socket says 172.21.15.66:46464) 2023-11-17T05:33:59.011 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:59.007+0000 7fa947fff700 1 -- 172.21.15.66:0/3756503059 learned_addr learned my addr 172.21.15.66:0/3756503059 (peer_addr_for_me v2:172.21.15.66:0/0) 2023-11-17T05:33:59.011 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:59.008+0000 7fa947fff700 1 -- 172.21.15.66:0/3756503059 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fa938005040 con 0x7fa9480f47c0 2023-11-17T05:33:59.012 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:59.008+0000 7fa947fff700 1 --2- 172.21.15.66:0/3756503059 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fa9480f47c0 0x7fa948106700 secure :-1 s=READY pgs=396 cs=0 l=1 rev1=1 rx=0x7fa938013510 tx=0x7fa93800b040).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:33:59.012 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:59.008+0000 7fa944ff9700 1 -- 172.21.15.66:0/3756503059 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7fa938009320 con 0x7fa9480f47c0 2023-11-17T05:33:59.012 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:59.008+0000 7fa94e8e6700 1 -- 172.21.15.66:0/3756503059 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fa948106df0 con 0x7fa9480f47c0 2023-11-17T05:33:59.012 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:59.008+0000 7fa94e8e6700 1 -- 172.21.15.66:0/3756503059 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fa948107280 con 0x7fa9480f47c0 2023-11-17T05:33:59.012 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:59.008+0000 7fa944ff9700 1 -- 172.21.15.66:0/3756503059 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(25 keys) v1 ==== 1074+0+0 (secure 0 0 0) 0x7fa938013f00 con 0x7fa9480f47c0 2023-11-17T05:33:59.012 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:59.008+0000 7fa944ff9700 1 -- 172.21.15.66:0/3756503059 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7fa938014d60 con 0x7fa9480f47c0 2023-11-17T05:33:59.012 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:59.009+0000 7fa944ff9700 1 -- 172.21.15.66:0/3756503059 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 17) v1 ==== 36269+0+0 (secure 0 0 0) 0x7fa9380095f0 con 0x7fa9480f47c0 2023-11-17T05:33:59.012 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:59.009+0000 7fa94e8e6700 1 -- 172.21.15.66:0/3756503059 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fa948045ad0 con 0x7fa9480f47c0 2023-11-17T05:33:59.012 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:59.009+0000 7fa944ff9700 1 --2- 172.21.15.66:0/3756503059 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fa93402ff60 0x7fa934032420 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:33:59.012 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:59.009+0000 7fa944ff9700 1 -- 172.21.15.66:0/3756503059 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(5..5 src has 1..5) v4 ==== 1270+0+0 (secure 0 0 0) 0x7fa938041010 con 0x7fa9480f47c0 2023-11-17T05:33:59.012 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:59.009+0000 7fa9477fe700 1 --2- 172.21.15.66:0/3756503059 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fa93402ff60 0x7fa934032420 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:33:59.013 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:59.010+0000 7fa9477fe700 1 --2- 172.21.15.66:0/3756503059 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fa93402ff60 0x7fa934032420 secure :-1 s=READY pgs=163 cs=0 l=1 rev1=1 rx=0x7fa9300096a0 tx=0x7fa930006b40).ready entity=mgr.14164 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:33:59.015 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:59.015+0000 7fa944ff9700 1 -- 172.21.15.66:0/3756503059 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+152212 (secure 0 0 0) 0x7fa938014420 con 0x7fa9480f47c0 2023-11-17T05:33:59.234 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:59.232+0000 7fa94e8e6700 1 -- 172.21.15.66:0/3756503059 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "mon dump", "format": "json"} v 0) v1 -- 0x7fa948108130 con 0x7fa9480f47c0 2023-11-17T05:33:59.234 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:59.233+0000 7fa944ff9700 1 -- 172.21.15.66:0/3756503059 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mon_command_ack([{"prefix": "mon dump", "format": "json"}]=0 dumped monmap epoch 1 v1) v1 ==== 95+0+690 (secure 0 0 0) 0x7fa938012980 con 0x7fa9480f47c0 2023-11-17T05:33:59.234 INFO:teuthology.orchestra.run.smithi066.stdout: 2023-11-17T05:33:59.235 INFO:teuthology.orchestra.run.smithi066.stdout:{"epoch":1,"fsid":"ed6470e2-8508-11ee-8dbe-212e2dc638e7","modified":"2023-11-17T05:20:13.934038Z","created":"2023-11-17T05:20:13.934038Z","min_mon_release":16,"min_mon_release_name":"pacific","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging"],"optional":[]},"mons":[{"rank":0,"name":"smithi040","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:3300","nonce":0},{"type":"v1","addr":"172.21.15.40:6789","nonce":0}]},"addr":"172.21.15.40:6789/0","public_addr":"172.21.15.40:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2023-11-17T05:33:59.236 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:59.235+0000 7fa94e8e6700 1 -- 172.21.15.66:0/3756503059 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fa93402ff60 msgr2=0x7fa934032420 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:33:59.236 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:59.235+0000 7fa94e8e6700 1 --2- 172.21.15.66:0/3756503059 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fa93402ff60 0x7fa934032420 secure :-1 s=READY pgs=163 cs=0 l=1 rev1=1 rx=0x7fa9300096a0 tx=0x7fa930006b40).stop 2023-11-17T05:33:59.236 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:59.235+0000 7fa94e8e6700 1 -- 172.21.15.66:0/3756503059 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fa9480f47c0 msgr2=0x7fa948106700 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:33:59.236 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:59.235+0000 7fa94e8e6700 1 --2- 172.21.15.66:0/3756503059 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fa9480f47c0 0x7fa948106700 secure :-1 s=READY pgs=396 cs=0 l=1 rev1=1 rx=0x7fa938013510 tx=0x7fa93800b040).stop 2023-11-17T05:33:59.236 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:59.235+0000 7fa94e8e6700 1 -- 172.21.15.66:0/3756503059 shutdown_connections 2023-11-17T05:33:59.236 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:59.235+0000 7fa94e8e6700 1 --2- 172.21.15.66:0/3756503059 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fa93402ff60 0x7fa934032420 unknown :-1 s=CLOSED pgs=163 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:33:59.237 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:59.235+0000 7fa94e8e6700 1 --2- 172.21.15.66:0/3756503059 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fa9480f47c0 0x7fa948106700 unknown :-1 s=CLOSED pgs=396 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:33:59.237 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:59.235+0000 7fa94e8e6700 1 -- 172.21.15.66:0/3756503059 >> 172.21.15.66:0/3756503059 conn(0x7fa9480f02e0 msgr2=0x7fa9480f0db0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:33:59.237 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:59.235+0000 7fa94e8e6700 1 -- 172.21.15.66:0/3756503059 shutdown_connections 2023-11-17T05:33:59.237 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:33:59.235+0000 7fa94e8e6700 1 -- 172.21.15.66:0/3756503059 wait complete. 2023-11-17T05:33:59.237 INFO:teuthology.orchestra.run.smithi066.stderr:dumped monmap epoch 1 2023-11-17T05:33:59.738 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:33:59 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:33:59.234806+0000 mon.smithi040 (mon.0) 312 : audit [DBG] from='client.? 172.21.15.66:0/3756503059' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-11-17T05:34:00.612 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2023-11-17T05:34:00.613 DEBUG:teuthology.orchestra.run.smithi066:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.4 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ed6470e2-8508-11ee-8dbe-212e2dc638e7 -- ceph mon dump -f json 2023-11-17T05:34:00.992 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:34:00 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:33:59.540769+0000 mgr.smithi040.ogcprl (mgr.14164) 395 : cluster [DBG] pgmap v379: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:34:01.848 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:01.847+0000 7f8c895bd700 1 -- 172.21.15.66:0/1913469053 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f8c84060280 msgr2=0x7f8c840f9010 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:34:01.849 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:01.847+0000 7f8c895bd700 1 --2- 172.21.15.66:0/1913469053 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f8c84060280 0x7f8c840f9010 secure :-1 s=READY pgs=397 cs=0 l=1 rev1=1 rx=0x7f8c78009fb0 tx=0x7f8c78009b30).stop 2023-11-17T05:34:01.849 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:01.847+0000 7f8c895bd700 1 -- 172.21.15.66:0/1913469053 shutdown_connections 2023-11-17T05:34:01.849 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:01.847+0000 7f8c895bd700 1 --2- 172.21.15.66:0/1913469053 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f8c84060280 0x7f8c840f9010 unknown :-1 s=CLOSED pgs=397 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:34:01.849 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:01.847+0000 7f8c895bd700 1 -- 172.21.15.66:0/1913469053 >> 172.21.15.66:0/1913469053 conn(0x7f8c840f0a90 msgr2=0x7f8c840f2eb0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:34:01.849 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:01.847+0000 7f8c895bd700 1 -- 172.21.15.66:0/1913469053 shutdown_connections 2023-11-17T05:34:01.849 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:01.847+0000 7f8c895bd700 1 -- 172.21.15.66:0/1913469053 wait complete. 2023-11-17T05:34:01.851 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:01.848+0000 7f8c895bd700 1 Processor -- start 2023-11-17T05:34:01.851 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:01.848+0000 7f8c895bd700 1 -- start start 2023-11-17T05:34:01.851 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:01.848+0000 7f8c895bd700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f8c84060280 0x7f8c840f6a90 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:34:01.851 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:01.848+0000 7f8c895bd700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f8c840f4f10 con 0x7f8c84060280 2023-11-17T05:34:01.852 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:01.848+0000 7f8c82ffd700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f8c84060280 0x7f8c840f6a90 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:34:01.852 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:01.848+0000 7f8c82ffd700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f8c84060280 0x7f8c840f6a90 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.66:46490/0 (socket says 172.21.15.66:46490) 2023-11-17T05:34:01.852 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:01.848+0000 7f8c82ffd700 1 -- 172.21.15.66:0/1698830220 learned_addr learned my addr 172.21.15.66:0/1698830220 (peer_addr_for_me v2:172.21.15.66:0/0) 2023-11-17T05:34:01.852 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:01.849+0000 7f8c82ffd700 1 -- 172.21.15.66:0/1698830220 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f8c78005040 con 0x7f8c84060280 2023-11-17T05:34:01.852 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:01.849+0000 7f8c82ffd700 1 --2- 172.21.15.66:0/1698830220 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f8c84060280 0x7f8c840f6a90 secure :-1 s=READY pgs=398 cs=0 l=1 rev1=1 rx=0x7f8c78013510 tx=0x7f8c7800b040).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:34:01.852 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:01.849+0000 7f8c6b7fe700 1 -- 172.21.15.66:0/1698830220 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f8c78009320 con 0x7f8c84060280 2023-11-17T05:34:01.852 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:01.849+0000 7f8c895bd700 1 -- 172.21.15.66:0/1698830220 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f8c840f50b0 con 0x7f8c84060280 2023-11-17T05:34:01.852 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:01.849+0000 7f8c6b7fe700 1 -- 172.21.15.66:0/1698830220 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(25 keys) v1 ==== 1074+0+0 (secure 0 0 0) 0x7f8c78013f00 con 0x7f8c84060280 2023-11-17T05:34:01.852 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:01.849+0000 7f8c6b7fe700 1 -- 172.21.15.66:0/1698830220 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f8c78014dc0 con 0x7f8c84060280 2023-11-17T05:34:01.852 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:01.849+0000 7f8c895bd700 1 -- 172.21.15.66:0/1698830220 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f8c840f5540 con 0x7f8c84060280 2023-11-17T05:34:01.853 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:01.850+0000 7f8c6b7fe700 1 -- 172.21.15.66:0/1698830220 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 17) v1 ==== 36269+0+0 (secure 0 0 0) 0x7f8c780095f0 con 0x7f8c84060280 2023-11-17T05:34:01.853 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:01.850+0000 7f8c6b7fe700 1 --2- 172.21.15.66:0/1698830220 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f8c6c02ff60 0x7f8c6c032420 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:34:01.853 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:01.850+0000 7f8c895bd700 1 -- 172.21.15.66:0/1698830220 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f8c84045ad0 con 0x7f8c84060280 2023-11-17T05:34:01.853 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:01.850+0000 7f8c6b7fe700 1 -- 172.21.15.66:0/1698830220 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(5..5 src has 1..5) v4 ==== 1270+0+0 (secure 0 0 0) 0x7f8c78041000 con 0x7f8c84060280 2023-11-17T05:34:01.853 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:01.850+0000 7f8c827fc700 1 --2- 172.21.15.66:0/1698830220 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f8c6c02ff60 0x7f8c6c032420 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:34:01.853 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:01.851+0000 7f8c827fc700 1 --2- 172.21.15.66:0/1698830220 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f8c6c02ff60 0x7f8c6c032420 secure :-1 s=READY pgs=164 cs=0 l=1 rev1=1 rx=0x7f8c740096a0 tx=0x7f8c74006b40).ready entity=mgr.14164 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:34:01.857 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:01.856+0000 7f8c6b7fe700 1 -- 172.21.15.66:0/1698830220 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+152212 (secure 0 0 0) 0x7f8c78014420 con 0x7f8c84060280 2023-11-17T05:34:01.988 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:34:01 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:34:01.541047+0000 mgr.smithi040.ogcprl (mgr.14164) 396 : cluster [DBG] pgmap v380: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:34:02.073 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:02.071+0000 7f8c895bd700 1 -- 172.21.15.66:0/1698830220 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "mon dump", "format": "json"} v 0) v1 -- 0x7f8c8405ea50 con 0x7f8c84060280 2023-11-17T05:34:02.074 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:02.072+0000 7f8c6b7fe700 1 -- 172.21.15.66:0/1698830220 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mon_command_ack([{"prefix": "mon dump", "format": "json"}]=0 dumped monmap epoch 1 v1) v1 ==== 95+0+690 (secure 0 0 0) 0x7f8c78040070 con 0x7f8c84060280 2023-11-17T05:34:02.074 INFO:teuthology.orchestra.run.smithi066.stdout: 2023-11-17T05:34:02.074 INFO:teuthology.orchestra.run.smithi066.stdout:{"epoch":1,"fsid":"ed6470e2-8508-11ee-8dbe-212e2dc638e7","modified":"2023-11-17T05:20:13.934038Z","created":"2023-11-17T05:20:13.934038Z","min_mon_release":16,"min_mon_release_name":"pacific","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging"],"optional":[]},"mons":[{"rank":0,"name":"smithi040","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:3300","nonce":0},{"type":"v1","addr":"172.21.15.40:6789","nonce":0}]},"addr":"172.21.15.40:6789/0","public_addr":"172.21.15.40:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2023-11-17T05:34:02.075 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:02.074+0000 7f8c895bd700 1 -- 172.21.15.66:0/1698830220 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f8c6c02ff60 msgr2=0x7f8c6c032420 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:34:02.076 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:02.074+0000 7f8c895bd700 1 --2- 172.21.15.66:0/1698830220 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f8c6c02ff60 0x7f8c6c032420 secure :-1 s=READY pgs=164 cs=0 l=1 rev1=1 rx=0x7f8c740096a0 tx=0x7f8c74006b40).stop 2023-11-17T05:34:02.076 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:02.074+0000 7f8c895bd700 1 -- 172.21.15.66:0/1698830220 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f8c84060280 msgr2=0x7f8c840f6a90 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:34:02.076 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:02.074+0000 7f8c895bd700 1 --2- 172.21.15.66:0/1698830220 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f8c84060280 0x7f8c840f6a90 secure :-1 s=READY pgs=398 cs=0 l=1 rev1=1 rx=0x7f8c78013510 tx=0x7f8c7800b040).stop 2023-11-17T05:34:02.076 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:02.074+0000 7f8c895bd700 1 -- 172.21.15.66:0/1698830220 shutdown_connections 2023-11-17T05:34:02.076 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:02.074+0000 7f8c895bd700 1 --2- 172.21.15.66:0/1698830220 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f8c6c02ff60 0x7f8c6c032420 unknown :-1 s=CLOSED pgs=164 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:34:02.076 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:02.074+0000 7f8c895bd700 1 --2- 172.21.15.66:0/1698830220 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f8c84060280 0x7f8c840f6a90 unknown :-1 s=CLOSED pgs=398 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:34:02.076 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:02.074+0000 7f8c895bd700 1 -- 172.21.15.66:0/1698830220 >> 172.21.15.66:0/1698830220 conn(0x7f8c840f0a90 msgr2=0x7f8c840f1520 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:34:02.076 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:02.074+0000 7f8c895bd700 1 -- 172.21.15.66:0/1698830220 shutdown_connections 2023-11-17T05:34:02.076 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:02.074+0000 7f8c895bd700 1 -- 172.21.15.66:0/1698830220 wait complete. 2023-11-17T05:34:02.076 INFO:teuthology.orchestra.run.smithi066.stderr:dumped monmap epoch 1 2023-11-17T05:34:02.989 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:34:02 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:34:02.073877+0000 mon.smithi040 (mon.0) 313 : audit [DBG] from='client.? 172.21.15.66:0/1698830220' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-11-17T05:34:03.504 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2023-11-17T05:34:03.504 DEBUG:teuthology.orchestra.run.smithi066:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.4 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ed6470e2-8508-11ee-8dbe-212e2dc638e7 -- ceph mon dump -f json 2023-11-17T05:34:03.989 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:34:03 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:34:03.541401+0000 mgr.smithi040.ogcprl (mgr.14164) 397 : cluster [DBG] pgmap v381: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:34:04.634 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:04.631+0000 7f69acdb6700 1 -- 172.21.15.66:0/1189270529 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f69a80f6510 msgr2=0x7f69a80f6930 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:34:04.634 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:04.631+0000 7f69acdb6700 1 --2- 172.21.15.66:0/1189270529 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f69a80f6510 0x7f69a80f6930 secure :-1 s=READY pgs=399 cs=0 l=1 rev1=1 rx=0x7f699c009fb0 tx=0x7f699c009b30).stop 2023-11-17T05:34:04.634 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:04.631+0000 7f69acdb6700 1 -- 172.21.15.66:0/1189270529 shutdown_connections 2023-11-17T05:34:04.634 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:04.631+0000 7f69acdb6700 1 --2- 172.21.15.66:0/1189270529 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f69a80f6510 0x7f69a80f6930 unknown :-1 s=CLOSED pgs=399 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:34:04.634 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:04.631+0000 7f69acdb6700 1 -- 172.21.15.66:0/1189270529 >> 172.21.15.66:0/1189270529 conn(0x7f69a80f1c70 msgr2=0x7f69a80f4090 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:34:04.634 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:04.631+0000 7f69acdb6700 1 -- 172.21.15.66:0/1189270529 shutdown_connections 2023-11-17T05:34:04.634 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:04.631+0000 7f69acdb6700 1 -- 172.21.15.66:0/1189270529 wait complete. 2023-11-17T05:34:04.634 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:04.631+0000 7f69acdb6700 1 Processor -- start 2023-11-17T05:34:04.634 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:04.632+0000 7f69acdb6700 1 -- start start 2023-11-17T05:34:04.635 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:04.632+0000 7f69acdb6700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f69a80f6510 0x7f69a8106730 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:34:04.635 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:04.632+0000 7f69acdb6700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f69a8106c20 con 0x7f69a80f6510 2023-11-17T05:34:04.635 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:04.632+0000 7f69a659c700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f69a80f6510 0x7f69a8106730 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:34:04.635 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:04.632+0000 7f69a659c700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f69a80f6510 0x7f69a8106730 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.66:44072/0 (socket says 172.21.15.66:44072) 2023-11-17T05:34:04.635 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:04.632+0000 7f69a659c700 1 -- 172.21.15.66:0/1569578551 learned_addr learned my addr 172.21.15.66:0/1569578551 (peer_addr_for_me v2:172.21.15.66:0/0) 2023-11-17T05:34:04.635 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:04.632+0000 7f69a659c700 1 -- 172.21.15.66:0/1569578551 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f699c005040 con 0x7f69a80f6510 2023-11-17T05:34:04.635 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:04.632+0000 7f69a659c700 1 --2- 172.21.15.66:0/1569578551 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f69a80f6510 0x7f69a8106730 secure :-1 s=READY pgs=400 cs=0 l=1 rev1=1 rx=0x7f699c013510 tx=0x7f699c00b040).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:34:04.635 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:04.632+0000 7f6996ffd700 1 -- 172.21.15.66:0/1569578551 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f699c009320 con 0x7f69a80f6510 2023-11-17T05:34:04.635 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:04.632+0000 7f6996ffd700 1 -- 172.21.15.66:0/1569578551 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(25 keys) v1 ==== 1074+0+0 (secure 0 0 0) 0x7f699c013f00 con 0x7f69a80f6510 2023-11-17T05:34:04.635 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:04.632+0000 7f69acdb6700 1 -- 172.21.15.66:0/1569578551 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f69a8106e20 con 0x7f69a80f6510 2023-11-17T05:34:04.635 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:04.632+0000 7f6996ffd700 1 -- 172.21.15.66:0/1569578551 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f699c014d90 con 0x7f69a80f6510 2023-11-17T05:34:04.635 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:04.633+0000 7f69acdb6700 1 -- 172.21.15.66:0/1569578551 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f69a81072b0 con 0x7f69a80f6510 2023-11-17T05:34:04.636 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:04.633+0000 7f6996ffd700 1 -- 172.21.15.66:0/1569578551 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 17) v1 ==== 36269+0+0 (secure 0 0 0) 0x7f699c0095f0 con 0x7f69a80f6510 2023-11-17T05:34:04.636 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:04.633+0000 7f69acdb6700 1 -- 172.21.15.66:0/1569578551 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f69a8045ad0 con 0x7f69a80f6510 2023-11-17T05:34:04.636 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:04.633+0000 7f6996ffd700 1 --2- 172.21.15.66:0/1569578551 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f699002ff60 0x7f6990032420 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:34:04.636 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:04.633+0000 7f6996ffd700 1 -- 172.21.15.66:0/1569578551 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(5..5 src has 1..5) v4 ==== 1270+0+0 (secure 0 0 0) 0x7f699c041080 con 0x7f69a80f6510 2023-11-17T05:34:04.637 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:04.634+0000 7f69a5d9b700 1 --2- 172.21.15.66:0/1569578551 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f699002ff60 0x7f6990032420 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:34:04.637 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:04.634+0000 7f69a5d9b700 1 --2- 172.21.15.66:0/1569578551 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f699002ff60 0x7f6990032420 secure :-1 s=READY pgs=165 cs=0 l=1 rev1=1 rx=0x7f69980096a0 tx=0x7f6998006b40).ready entity=mgr.14164 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:34:04.640 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:04.639+0000 7f6996ffd700 1 -- 172.21.15.66:0/1569578551 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+152212 (secure 0 0 0) 0x7f699c014420 con 0x7f69a80f6510 2023-11-17T05:34:04.861 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:04.859+0000 7f69acdb6700 1 -- 172.21.15.66:0/1569578551 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "mon dump", "format": "json"} v 0) v1 -- 0x7f69a80fbf60 con 0x7f69a80f6510 2023-11-17T05:34:04.861 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:04.860+0000 7f6996ffd700 1 -- 172.21.15.66:0/1569578551 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mon_command_ack([{"prefix": "mon dump", "format": "json"}]=0 dumped monmap epoch 1 v1) v1 ==== 95+0+690 (secure 0 0 0) 0x7f699c010230 con 0x7f69a80f6510 2023-11-17T05:34:04.862 INFO:teuthology.orchestra.run.smithi066.stdout: 2023-11-17T05:34:04.862 INFO:teuthology.orchestra.run.smithi066.stdout:{"epoch":1,"fsid":"ed6470e2-8508-11ee-8dbe-212e2dc638e7","modified":"2023-11-17T05:20:13.934038Z","created":"2023-11-17T05:20:13.934038Z","min_mon_release":16,"min_mon_release_name":"pacific","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging"],"optional":[]},"mons":[{"rank":0,"name":"smithi040","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:3300","nonce":0},{"type":"v1","addr":"172.21.15.40:6789","nonce":0}]},"addr":"172.21.15.40:6789/0","public_addr":"172.21.15.40:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2023-11-17T05:34:04.863 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:04.862+0000 7f69acdb6700 1 -- 172.21.15.66:0/1569578551 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f699002ff60 msgr2=0x7f6990032420 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:34:04.863 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:04.862+0000 7f69acdb6700 1 --2- 172.21.15.66:0/1569578551 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f699002ff60 0x7f6990032420 secure :-1 s=READY pgs=165 cs=0 l=1 rev1=1 rx=0x7f69980096a0 tx=0x7f6998006b40).stop 2023-11-17T05:34:04.863 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:04.862+0000 7f69acdb6700 1 -- 172.21.15.66:0/1569578551 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f69a80f6510 msgr2=0x7f69a8106730 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:34:04.863 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:04.862+0000 7f69acdb6700 1 --2- 172.21.15.66:0/1569578551 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f69a80f6510 0x7f69a8106730 secure :-1 s=READY pgs=400 cs=0 l=1 rev1=1 rx=0x7f699c013510 tx=0x7f699c00b040).stop 2023-11-17T05:34:04.863 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:04.862+0000 7f69acdb6700 1 -- 172.21.15.66:0/1569578551 shutdown_connections 2023-11-17T05:34:04.863 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:04.862+0000 7f69acdb6700 1 --2- 172.21.15.66:0/1569578551 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f699002ff60 0x7f6990032420 unknown :-1 s=CLOSED pgs=165 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:34:04.864 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:04.862+0000 7f69acdb6700 1 --2- 172.21.15.66:0/1569578551 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f69a80f6510 0x7f69a8106730 unknown :-1 s=CLOSED pgs=400 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:34:04.864 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:04.862+0000 7f69acdb6700 1 -- 172.21.15.66:0/1569578551 >> 172.21.15.66:0/1569578551 conn(0x7f69a80f1c70 msgr2=0x7f69a80f2760 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:34:04.864 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:04.862+0000 7f69acdb6700 1 -- 172.21.15.66:0/1569578551 shutdown_connections 2023-11-17T05:34:04.864 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:04.862+0000 7f69acdb6700 1 -- 172.21.15.66:0/1569578551 wait complete. 2023-11-17T05:34:04.864 INFO:teuthology.orchestra.run.smithi066.stderr:dumped monmap epoch 1 2023-11-17T05:34:05.988 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:34:05 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:34:04.861838+0000 mon.smithi040 (mon.0) 314 : audit [DBG] from='client.? 172.21.15.66:0/1569578551' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-11-17T05:34:06.240 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2023-11-17T05:34:06.240 DEBUG:teuthology.orchestra.run.smithi066:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.4 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ed6470e2-8508-11ee-8dbe-212e2dc638e7 -- ceph mon dump -f json 2023-11-17T05:34:06.988 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:34:06 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:34:05.541725+0000 mgr.smithi040.ogcprl (mgr.14164) 398 : cluster [DBG] pgmap v382: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:34:07.406 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:07.405+0000 7f747da08700 1 -- 172.21.15.66:0/2159867222 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f74780f5410 msgr2=0x7f74780f5830 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:34:07.406 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:07.405+0000 7f747da08700 1 --2- 172.21.15.66:0/2159867222 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f74780f5410 0x7f74780f5830 secure :-1 s=READY pgs=401 cs=0 l=1 rev1=1 rx=0x7f7468009fb0 tx=0x7f7468009b30).stop 2023-11-17T05:34:07.406 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:07.405+0000 7f747da08700 1 -- 172.21.15.66:0/2159867222 shutdown_connections 2023-11-17T05:34:07.406 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:07.405+0000 7f747da08700 1 --2- 172.21.15.66:0/2159867222 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f74780f5410 0x7f74780f5830 unknown :-1 s=CLOSED pgs=401 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:34:07.406 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:07.405+0000 7f747da08700 1 -- 172.21.15.66:0/2159867222 >> 172.21.15.66:0/2159867222 conn(0x7f74780f0b70 msgr2=0x7f74780f2f90 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:34:07.406 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:07.406+0000 7f747da08700 1 -- 172.21.15.66:0/2159867222 shutdown_connections 2023-11-17T05:34:07.406 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:07.406+0000 7f747da08700 1 -- 172.21.15.66:0/2159867222 wait complete. 2023-11-17T05:34:07.408 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:07.406+0000 7f747da08700 1 Processor -- start 2023-11-17T05:34:07.408 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:07.406+0000 7f747da08700 1 -- start start 2023-11-17T05:34:07.408 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:07.406+0000 7f747da08700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f74780f5410 0x7f7478106720 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:34:07.408 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:07.406+0000 7f747da08700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f7478106c10 con 0x7f74780f5410 2023-11-17T05:34:07.408 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:07.407+0000 7f7476ffd700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f74780f5410 0x7f7478106720 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:34:07.408 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:07.407+0000 7f7476ffd700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f74780f5410 0x7f7478106720 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.66:44088/0 (socket says 172.21.15.66:44088) 2023-11-17T05:34:07.408 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:07.407+0000 7f7476ffd700 1 -- 172.21.15.66:0/3482677615 learned_addr learned my addr 172.21.15.66:0/3482677615 (peer_addr_for_me v2:172.21.15.66:0/0) 2023-11-17T05:34:07.408 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:07.407+0000 7f7476ffd700 1 -- 172.21.15.66:0/3482677615 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f7468005040 con 0x7f74780f5410 2023-11-17T05:34:07.408 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:07.407+0000 7f7476ffd700 1 --2- 172.21.15.66:0/3482677615 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f74780f5410 0x7f7478106720 secure :-1 s=READY pgs=402 cs=0 l=1 rev1=1 rx=0x7f7468013510 tx=0x7f746800b040).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:34:07.409 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:07.407+0000 7f746f7fe700 1 -- 172.21.15.66:0/3482677615 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f7468009320 con 0x7f74780f5410 2023-11-17T05:34:07.409 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:07.407+0000 7f747da08700 1 -- 172.21.15.66:0/3482677615 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f7478106e10 con 0x7f74780f5410 2023-11-17T05:34:07.409 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:07.407+0000 7f746f7fe700 1 -- 172.21.15.66:0/3482677615 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(25 keys) v1 ==== 1074+0+0 (secure 0 0 0) 0x7f7468013f00 con 0x7f74780f5410 2023-11-17T05:34:07.409 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:07.407+0000 7f746f7fe700 1 -- 172.21.15.66:0/3482677615 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f7468014d90 con 0x7f74780f5410 2023-11-17T05:34:07.409 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:07.407+0000 7f747da08700 1 -- 172.21.15.66:0/3482677615 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f74781072a0 con 0x7f74780f5410 2023-11-17T05:34:07.410 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:07.408+0000 7f746f7fe700 1 -- 172.21.15.66:0/3482677615 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 17) v1 ==== 36269+0+0 (secure 0 0 0) 0x7f74680095f0 con 0x7f74780f5410 2023-11-17T05:34:07.410 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:07.408+0000 7f747da08700 1 -- 172.21.15.66:0/3482677615 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f7478045ad0 con 0x7f74780f5410 2023-11-17T05:34:07.411 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:07.408+0000 7f746f7fe700 1 --2- 172.21.15.66:0/3482677615 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f746402ff60 0x7f7464032420 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:34:07.411 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:07.408+0000 7f746f7fe700 1 -- 172.21.15.66:0/3482677615 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(5..5 src has 1..5) v4 ==== 1270+0+0 (secure 0 0 0) 0x7f7468041000 con 0x7f74780f5410 2023-11-17T05:34:07.411 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:07.409+0000 7f74767fc700 1 --2- 172.21.15.66:0/3482677615 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f746402ff60 0x7f7464032420 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:34:07.411 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:07.409+0000 7f74767fc700 1 --2- 172.21.15.66:0/3482677615 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f746402ff60 0x7f7464032420 secure :-1 s=READY pgs=166 cs=0 l=1 rev1=1 rx=0x7f74600096a0 tx=0x7f7460006b40).ready entity=mgr.14164 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:34:07.413 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:07.413+0000 7f746f7fe700 1 -- 172.21.15.66:0/3482677615 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+152212 (secure 0 0 0) 0x7f7468014420 con 0x7f74780f5410 2023-11-17T05:34:07.633 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:07.633+0000 7f747da08700 1 -- 172.21.15.66:0/3482677615 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "mon dump", "format": "json"} v 0) v1 -- 0x7f7478108150 con 0x7f74780f5410 2023-11-17T05:34:07.634 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:07.634+0000 7f746f7fe700 1 -- 172.21.15.66:0/3482677615 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mon_command_ack([{"prefix": "mon dump", "format": "json"}]=0 dumped monmap epoch 1 v1) v1 ==== 95+0+690 (secure 0 0 0) 0x7f746801d7f0 con 0x7f74780f5410 2023-11-17T05:34:07.635 INFO:teuthology.orchestra.run.smithi066.stdout: 2023-11-17T05:34:07.635 INFO:teuthology.orchestra.run.smithi066.stdout:{"epoch":1,"fsid":"ed6470e2-8508-11ee-8dbe-212e2dc638e7","modified":"2023-11-17T05:20:13.934038Z","created":"2023-11-17T05:20:13.934038Z","min_mon_release":16,"min_mon_release_name":"pacific","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging"],"optional":[]},"mons":[{"rank":0,"name":"smithi040","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:3300","nonce":0},{"type":"v1","addr":"172.21.15.40:6789","nonce":0}]},"addr":"172.21.15.40:6789/0","public_addr":"172.21.15.40:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2023-11-17T05:34:07.636 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:07.636+0000 7f747da08700 1 -- 172.21.15.66:0/3482677615 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f746402ff60 msgr2=0x7f7464032420 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:34:07.636 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:07.636+0000 7f747da08700 1 --2- 172.21.15.66:0/3482677615 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f746402ff60 0x7f7464032420 secure :-1 s=READY pgs=166 cs=0 l=1 rev1=1 rx=0x7f74600096a0 tx=0x7f7460006b40).stop 2023-11-17T05:34:07.636 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:07.636+0000 7f747da08700 1 -- 172.21.15.66:0/3482677615 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f74780f5410 msgr2=0x7f7478106720 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:34:07.636 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:07.636+0000 7f747da08700 1 --2- 172.21.15.66:0/3482677615 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f74780f5410 0x7f7478106720 secure :-1 s=READY pgs=402 cs=0 l=1 rev1=1 rx=0x7f7468013510 tx=0x7f746800b040).stop 2023-11-17T05:34:07.637 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:07.636+0000 7f747da08700 1 -- 172.21.15.66:0/3482677615 shutdown_connections 2023-11-17T05:34:07.637 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:07.636+0000 7f747da08700 1 --2- 172.21.15.66:0/3482677615 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f746402ff60 0x7f7464032420 unknown :-1 s=CLOSED pgs=166 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:34:07.637 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:07.636+0000 7f747da08700 1 --2- 172.21.15.66:0/3482677615 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f74780f5410 0x7f7478106720 unknown :-1 s=CLOSED pgs=402 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:34:07.637 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:07.636+0000 7f747da08700 1 -- 172.21.15.66:0/3482677615 >> 172.21.15.66:0/3482677615 conn(0x7f74780f0b70 msgr2=0x7f747805fe20 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:34:07.637 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:07.636+0000 7f747da08700 1 -- 172.21.15.66:0/3482677615 shutdown_connections 2023-11-17T05:34:07.637 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:07.636+0000 7f747da08700 1 -- 172.21.15.66:0/3482677615 wait complete. 2023-11-17T05:34:07.637 INFO:teuthology.orchestra.run.smithi066.stderr:dumped monmap epoch 1 2023-11-17T05:34:07.989 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:34:07 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:34:07.542099+0000 mgr.smithi040.ogcprl (mgr.14164) 399 : cluster [DBG] pgmap v383: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:34:08.989 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:34:08 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:34:07.634625+0000 mon.smithi040 (mon.0) 315 : audit [DBG] from='client.? 172.21.15.66:0/3482677615' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-11-17T05:34:16.166 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2023-11-17T05:34:16.167 DEBUG:teuthology.orchestra.run.smithi066:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.4 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ed6470e2-8508-11ee-8dbe-212e2dc638e7 -- ceph mon dump -f json 2023-11-17T05:34:16.168 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:34:10 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:34:09.542435+0000 mgr.smithi040.ogcprl (mgr.14164) 400 : cluster [DBG] pgmap v384: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:34:16.168 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:34:11 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:34:11.322123+0000 mon.smithi040 (mon.0) 316 : audit [INF] from='mgr.14164 172.21.15.40:0/2203032451' entity='mgr.smithi040.ogcprl' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi040.ogcprl/mirror_snapshot_schedule"}]: dispatch 2023-11-17T05:34:16.168 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:34:11 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:34:11.322380+0000 mon.smithi040 (mon.0) 317 : audit [INF] from='mgr.14164 172.21.15.40:0/2203032451' entity='mgr.smithi040.ogcprl' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi040.ogcprl/trash_purge_schedule"}]: dispatch 2023-11-17T05:34:16.169 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:34:12 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:34:11.542735+0000 mgr.smithi040.ogcprl (mgr.14164) 401 : cluster [DBG] pgmap v385: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:34:16.169 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:34:13 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:34:13.543085+0000 mgr.smithi040.ogcprl (mgr.14164) 402 : cluster [DBG] pgmap v386: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:34:16.169 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:34:15 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:34:15.543449+0000 mgr.smithi040.ogcprl (mgr.14164) 403 : cluster [DBG] pgmap v387: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:34:17.341 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:17.338+0000 7fb9cbeba700 1 -- 172.21.15.66:0/1695951504 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fb9c40f5410 msgr2=0x7fb9c40f5830 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:34:17.341 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:17.338+0000 7fb9cbeba700 1 --2- 172.21.15.66:0/1695951504 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fb9c40f5410 0x7fb9c40f5830 secure :-1 s=READY pgs=403 cs=0 l=1 rev1=1 rx=0x7fb9b4009fb0 tx=0x7fb9b4009b30).stop 2023-11-17T05:34:17.341 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:17.338+0000 7fb9cbeba700 1 -- 172.21.15.66:0/1695951504 shutdown_connections 2023-11-17T05:34:17.341 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:17.338+0000 7fb9cbeba700 1 --2- 172.21.15.66:0/1695951504 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fb9c40f5410 0x7fb9c40f5830 unknown :-1 s=CLOSED pgs=403 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:34:17.341 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:17.338+0000 7fb9cbeba700 1 -- 172.21.15.66:0/1695951504 >> 172.21.15.66:0/1695951504 conn(0x7fb9c40f0b70 msgr2=0x7fb9c40f2f90 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:34:17.341 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:17.339+0000 7fb9cbeba700 1 -- 172.21.15.66:0/1695951504 shutdown_connections 2023-11-17T05:34:17.341 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:17.339+0000 7fb9cbeba700 1 -- 172.21.15.66:0/1695951504 wait complete. 2023-11-17T05:34:17.341 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:17.339+0000 7fb9cbeba700 1 Processor -- start 2023-11-17T05:34:17.341 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:17.339+0000 7fb9cbeba700 1 -- start start 2023-11-17T05:34:17.342 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:17.339+0000 7fb9cbeba700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fb9c40f5410 0x7fb9c41044b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:34:17.342 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:17.339+0000 7fb9cbeba700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fb9c41049a0 con 0x7fb9c40f5410 2023-11-17T05:34:17.342 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:17.340+0000 7fb9c9c56700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fb9c40f5410 0x7fb9c41044b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:34:17.342 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:17.340+0000 7fb9c9c56700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fb9c40f5410 0x7fb9c41044b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.66:43938/0 (socket says 172.21.15.66:43938) 2023-11-17T05:34:17.342 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:17.340+0000 7fb9c9c56700 1 -- 172.21.15.66:0/157709589 learned_addr learned my addr 172.21.15.66:0/157709589 (peer_addr_for_me v2:172.21.15.66:0/0) 2023-11-17T05:34:17.342 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:17.340+0000 7fb9c9c56700 1 -- 172.21.15.66:0/157709589 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fb9b4005040 con 0x7fb9c40f5410 2023-11-17T05:34:17.342 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:17.340+0000 7fb9c9c56700 1 --2- 172.21.15.66:0/157709589 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fb9c40f5410 0x7fb9c41044b0 secure :-1 s=READY pgs=404 cs=0 l=1 rev1=1 rx=0x7fb9b4013510 tx=0x7fb9b400b040).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:34:17.342 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:17.340+0000 7fb9ba7fc700 1 -- 172.21.15.66:0/157709589 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7fb9b4009320 con 0x7fb9c40f5410 2023-11-17T05:34:17.342 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:17.340+0000 7fb9ba7fc700 1 -- 172.21.15.66:0/157709589 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(25 keys) v1 ==== 1074+0+0 (secure 0 0 0) 0x7fb9b4013f00 con 0x7fb9c40f5410 2023-11-17T05:34:17.342 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:17.340+0000 7fb9cbeba700 1 -- 172.21.15.66:0/157709589 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fb9c4104ba0 con 0x7fb9c40f5410 2023-11-17T05:34:17.342 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:17.340+0000 7fb9ba7fc700 1 -- 172.21.15.66:0/157709589 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7fb9b4014d90 con 0x7fb9c40f5410 2023-11-17T05:34:17.343 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:17.340+0000 7fb9cbeba700 1 -- 172.21.15.66:0/157709589 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fb9c4105030 con 0x7fb9c40f5410 2023-11-17T05:34:17.343 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:17.341+0000 7fb9ba7fc700 1 -- 172.21.15.66:0/157709589 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 17) v1 ==== 36269+0+0 (secure 0 0 0) 0x7fb9b40095f0 con 0x7fb9c40f5410 2023-11-17T05:34:17.344 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:17.341+0000 7fb9cbeba700 1 -- 172.21.15.66:0/157709589 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fb9c4045ad0 con 0x7fb9c40f5410 2023-11-17T05:34:17.344 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:17.341+0000 7fb9ba7fc700 1 --2- 172.21.15.66:0/157709589 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fb9b002ff60 0x7fb9b0032420 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:34:17.344 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:17.341+0000 7fb9ba7fc700 1 -- 172.21.15.66:0/157709589 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(5..5 src has 1..5) v4 ==== 1270+0+0 (secure 0 0 0) 0x7fb9b40410c0 con 0x7fb9c40f5410 2023-11-17T05:34:17.344 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:17.342+0000 7fb9c9455700 1 --2- 172.21.15.66:0/157709589 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fb9b002ff60 0x7fb9b0032420 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:34:17.344 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:17.342+0000 7fb9c9455700 1 --2- 172.21.15.66:0/157709589 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fb9b002ff60 0x7fb9b0032420 secure :-1 s=READY pgs=167 cs=0 l=1 rev1=1 rx=0x7fb9c00096a0 tx=0x7fb9c0006b40).ready entity=mgr.14164 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:34:17.346 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:17.346+0000 7fb9ba7fc700 1 -- 172.21.15.66:0/157709589 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+152212 (secure 0 0 0) 0x7fb9b4014420 con 0x7fb9c40f5410 2023-11-17T05:34:17.570 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:17.569+0000 7fb9cbeba700 1 -- 172.21.15.66:0/157709589 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "mon dump", "format": "json"} v 0) v1 -- 0x7fb9c40fae60 con 0x7fb9c40f5410 2023-11-17T05:34:17.570 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:17.570+0000 7fb9ba7fc700 1 -- 172.21.15.66:0/157709589 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mon_command_ack([{"prefix": "mon dump", "format": "json"}]=0 dumped monmap epoch 1 v1) v1 ==== 95+0+690 (secure 0 0 0) 0x7fb9b4010230 con 0x7fb9c40f5410 2023-11-17T05:34:17.570 INFO:teuthology.orchestra.run.smithi066.stdout: 2023-11-17T05:34:17.571 INFO:teuthology.orchestra.run.smithi066.stdout:{"epoch":1,"fsid":"ed6470e2-8508-11ee-8dbe-212e2dc638e7","modified":"2023-11-17T05:20:13.934038Z","created":"2023-11-17T05:20:13.934038Z","min_mon_release":16,"min_mon_release_name":"pacific","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging"],"optional":[]},"mons":[{"rank":0,"name":"smithi040","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:3300","nonce":0},{"type":"v1","addr":"172.21.15.40:6789","nonce":0}]},"addr":"172.21.15.40:6789/0","public_addr":"172.21.15.40:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2023-11-17T05:34:17.572 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:17.572+0000 7fb9cbeba700 1 -- 172.21.15.66:0/157709589 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fb9b002ff60 msgr2=0x7fb9b0032420 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:34:17.572 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:17.572+0000 7fb9cbeba700 1 --2- 172.21.15.66:0/157709589 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fb9b002ff60 0x7fb9b0032420 secure :-1 s=READY pgs=167 cs=0 l=1 rev1=1 rx=0x7fb9c00096a0 tx=0x7fb9c0006b40).stop 2023-11-17T05:34:17.572 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:17.572+0000 7fb9cbeba700 1 -- 172.21.15.66:0/157709589 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fb9c40f5410 msgr2=0x7fb9c41044b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:34:17.572 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:17.572+0000 7fb9cbeba700 1 --2- 172.21.15.66:0/157709589 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fb9c40f5410 0x7fb9c41044b0 secure :-1 s=READY pgs=404 cs=0 l=1 rev1=1 rx=0x7fb9b4013510 tx=0x7fb9b400b040).stop 2023-11-17T05:34:17.572 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:17.572+0000 7fb9cbeba700 1 -- 172.21.15.66:0/157709589 shutdown_connections 2023-11-17T05:34:17.572 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:17.572+0000 7fb9cbeba700 1 --2- 172.21.15.66:0/157709589 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fb9b002ff60 0x7fb9b0032420 unknown :-1 s=CLOSED pgs=167 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:34:17.572 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:17.572+0000 7fb9cbeba700 1 --2- 172.21.15.66:0/157709589 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fb9c40f5410 0x7fb9c41044b0 unknown :-1 s=CLOSED pgs=404 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:34:17.573 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:17.572+0000 7fb9cbeba700 1 -- 172.21.15.66:0/157709589 >> 172.21.15.66:0/157709589 conn(0x7fb9c40f0b70 msgr2=0x7fb9c405fea0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:34:17.573 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:17.572+0000 7fb9cbeba700 1 -- 172.21.15.66:0/157709589 shutdown_connections 2023-11-17T05:34:17.573 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:17.572+0000 7fb9cbeba700 1 -- 172.21.15.66:0/157709589 wait complete. 2023-11-17T05:34:17.573 INFO:teuthology.orchestra.run.smithi066.stderr:dumped monmap epoch 1 2023-11-17T05:34:17.988 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:34:17 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:34:17.543892+0000 mgr.smithi040.ogcprl (mgr.14164) 404 : cluster [DBG] pgmap v388: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:34:17.989 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:34:17 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:34:17.570714+0000 mon.smithi040 (mon.0) 318 : audit [DBG] from='client.? 172.21.15.66:0/157709589' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-11-17T05:34:18.979 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2023-11-17T05:34:18.979 DEBUG:teuthology.orchestra.run.smithi066:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.4 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ed6470e2-8508-11ee-8dbe-212e2dc638e7 -- ceph mon dump -f json 2023-11-17T05:34:20.216 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:20.208+0000 7f8965b41700 1 -- 172.21.15.66:0/1713059361 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f89600f5410 msgr2=0x7f89600f5830 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:34:20.216 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:20.208+0000 7f8965b41700 1 --2- 172.21.15.66:0/1713059361 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f89600f5410 0x7f89600f5830 secure :-1 s=READY pgs=405 cs=0 l=1 rev1=1 rx=0x7f8950009fb0 tx=0x7f8950009b30).stop 2023-11-17T05:34:20.216 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:20.208+0000 7f8965b41700 1 -- 172.21.15.66:0/1713059361 shutdown_connections 2023-11-17T05:34:20.216 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:20.208+0000 7f8965b41700 1 --2- 172.21.15.66:0/1713059361 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f89600f5410 0x7f89600f5830 unknown :-1 s=CLOSED pgs=405 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:34:20.217 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:20.208+0000 7f8965b41700 1 -- 172.21.15.66:0/1713059361 >> 172.21.15.66:0/1713059361 conn(0x7f89600f0b70 msgr2=0x7f89600f2f90 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:34:20.217 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:20.208+0000 7f8965b41700 1 -- 172.21.15.66:0/1713059361 shutdown_connections 2023-11-17T05:34:20.217 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:20.208+0000 7f8965b41700 1 -- 172.21.15.66:0/1713059361 wait complete. 2023-11-17T05:34:20.217 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:20.209+0000 7f8965b41700 1 Processor -- start 2023-11-17T05:34:20.217 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:20.209+0000 7f8965b41700 1 -- start start 2023-11-17T05:34:20.217 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:20.209+0000 7f8965b41700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f89600f5410 0x7f8960109450 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:34:20.218 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:20.209+0000 7f8965b41700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f8960060220 con 0x7f89600f5410 2023-11-17T05:34:20.218 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:20.209+0000 7f895f7fe700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f89600f5410 0x7f8960109450 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:34:20.218 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:20.209+0000 7f895f7fe700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f89600f5410 0x7f8960109450 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.66:43954/0 (socket says 172.21.15.66:43954) 2023-11-17T05:34:20.218 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:20.209+0000 7f895f7fe700 1 -- 172.21.15.66:0/2449863892 learned_addr learned my addr 172.21.15.66:0/2449863892 (peer_addr_for_me v2:172.21.15.66:0/0) 2023-11-17T05:34:20.218 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:20.210+0000 7f895f7fe700 1 -- 172.21.15.66:0/2449863892 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f8950005040 con 0x7f89600f5410 2023-11-17T05:34:20.218 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:20.210+0000 7f895f7fe700 1 --2- 172.21.15.66:0/2449863892 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f89600f5410 0x7f8960109450 secure :-1 s=READY pgs=406 cs=0 l=1 rev1=1 rx=0x7f8950013510 tx=0x7f895000b040).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:34:20.218 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:20.210+0000 7f8957fff700 1 -- 172.21.15.66:0/2449863892 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f8950009320 con 0x7f89600f5410 2023-11-17T05:34:20.218 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:20.210+0000 7f8965b41700 1 -- 172.21.15.66:0/2449863892 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f8960060420 con 0x7f89600f5410 2023-11-17T05:34:20.218 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:20.210+0000 7f8957fff700 1 -- 172.21.15.66:0/2449863892 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(25 keys) v1 ==== 1074+0+0 (secure 0 0 0) 0x7f8950013f00 con 0x7f89600f5410 2023-11-17T05:34:20.218 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:20.210+0000 7f8957fff700 1 -- 172.21.15.66:0/2449863892 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f8950014d90 con 0x7f89600f5410 2023-11-17T05:34:20.219 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:20.210+0000 7f8965b41700 1 -- 172.21.15.66:0/2449863892 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f8960109b00 con 0x7f89600f5410 2023-11-17T05:34:20.219 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:20.211+0000 7f8957fff700 1 -- 172.21.15.66:0/2449863892 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 17) v1 ==== 36269+0+0 (secure 0 0 0) 0x7f89500095f0 con 0x7f89600f5410 2023-11-17T05:34:20.219 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:20.211+0000 7f8965b41700 1 -- 172.21.15.66:0/2449863892 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f8960059100 con 0x7f89600f5410 2023-11-17T05:34:20.219 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:20.211+0000 7f8957fff700 1 --2- 172.21.15.66:0/2449863892 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f894c02ff60 0x7f894c032420 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:34:20.219 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:20.211+0000 7f8957fff700 1 -- 172.21.15.66:0/2449863892 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(5..5 src has 1..5) v4 ==== 1270+0+0 (secure 0 0 0) 0x7f8950041000 con 0x7f89600f5410 2023-11-17T05:34:20.219 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:20.211+0000 7f895effd700 1 --2- 172.21.15.66:0/2449863892 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f894c02ff60 0x7f894c032420 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:34:20.219 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:20.212+0000 7f895effd700 1 --2- 172.21.15.66:0/2449863892 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f894c02ff60 0x7f894c032420 secure :-1 s=READY pgs=168 cs=0 l=1 rev1=1 rx=0x7f89480096a0 tx=0x7f8948006b40).ready entity=mgr.14164 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:34:20.220 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:20.216+0000 7f8957fff700 1 -- 172.21.15.66:0/2449863892 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+152212 (secure 0 0 0) 0x7f8950010230 con 0x7f89600f5410 2023-11-17T05:34:20.435 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:20.433+0000 7f8965b41700 1 -- 172.21.15.66:0/2449863892 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "mon dump", "format": "json"} v 0) v1 -- 0x7f8960045ad0 con 0x7f89600f5410 2023-11-17T05:34:20.435 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:20.433+0000 7f8957fff700 1 -- 172.21.15.66:0/2449863892 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mon_command_ack([{"prefix": "mon dump", "format": "json"}]=0 dumped monmap epoch 1 v1) v1 ==== 95+0+690 (secure 0 0 0) 0x7f8950010230 con 0x7f89600f5410 2023-11-17T05:34:20.435 INFO:teuthology.orchestra.run.smithi066.stdout: 2023-11-17T05:34:20.435 INFO:teuthology.orchestra.run.smithi066.stdout:{"epoch":1,"fsid":"ed6470e2-8508-11ee-8dbe-212e2dc638e7","modified":"2023-11-17T05:20:13.934038Z","created":"2023-11-17T05:20:13.934038Z","min_mon_release":16,"min_mon_release_name":"pacific","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging"],"optional":[]},"mons":[{"rank":0,"name":"smithi040","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:3300","nonce":0},{"type":"v1","addr":"172.21.15.40:6789","nonce":0}]},"addr":"172.21.15.40:6789/0","public_addr":"172.21.15.40:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2023-11-17T05:34:20.436 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:20.435+0000 7f8965b41700 1 -- 172.21.15.66:0/2449863892 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f894c02ff60 msgr2=0x7f894c032420 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:34:20.436 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:20.435+0000 7f8965b41700 1 --2- 172.21.15.66:0/2449863892 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f894c02ff60 0x7f894c032420 secure :-1 s=READY pgs=168 cs=0 l=1 rev1=1 rx=0x7f89480096a0 tx=0x7f8948006b40).stop 2023-11-17T05:34:20.436 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:20.435+0000 7f8965b41700 1 -- 172.21.15.66:0/2449863892 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f89600f5410 msgr2=0x7f8960109450 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:34:20.436 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:20.435+0000 7f8965b41700 1 --2- 172.21.15.66:0/2449863892 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f89600f5410 0x7f8960109450 secure :-1 s=READY pgs=406 cs=0 l=1 rev1=1 rx=0x7f8950013510 tx=0x7f895000b040).stop 2023-11-17T05:34:20.436 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:20.435+0000 7f8965b41700 1 -- 172.21.15.66:0/2449863892 shutdown_connections 2023-11-17T05:34:20.436 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:20.435+0000 7f8965b41700 1 --2- 172.21.15.66:0/2449863892 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f894c02ff60 0x7f894c032420 unknown :-1 s=CLOSED pgs=168 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:34:20.437 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:20.435+0000 7f8965b41700 1 --2- 172.21.15.66:0/2449863892 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f89600f5410 0x7f8960109450 unknown :-1 s=CLOSED pgs=406 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:34:20.437 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:20.435+0000 7f8965b41700 1 -- 172.21.15.66:0/2449863892 >> 172.21.15.66:0/2449863892 conn(0x7f89600f0b70 msgr2=0x7f89600f1660 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:34:20.437 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:20.435+0000 7f8965b41700 1 -- 172.21.15.66:0/2449863892 shutdown_connections 2023-11-17T05:34:20.437 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:20.435+0000 7f8965b41700 1 -- 172.21.15.66:0/2449863892 wait complete. 2023-11-17T05:34:20.437 INFO:teuthology.orchestra.run.smithi066.stderr:dumped monmap epoch 1 2023-11-17T05:34:20.988 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:34:20 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:34:19.544291+0000 mgr.smithi040.ogcprl (mgr.14164) 405 : cluster [DBG] pgmap v389: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:34:20.989 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:34:20 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:34:20.433940+0000 mon.smithi040 (mon.0) 319 : audit [DBG] from='client.? 172.21.15.66:0/2449863892' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-11-17T05:34:21.819 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2023-11-17T05:34:21.819 DEBUG:teuthology.orchestra.run.smithi066:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.4 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ed6470e2-8508-11ee-8dbe-212e2dc638e7 -- ceph mon dump -f json 2023-11-17T05:34:21.988 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:34:21 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:34:21.544748+0000 mgr.smithi040.ogcprl (mgr.14164) 406 : cluster [DBG] pgmap v390: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:34:23.067 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:23.066+0000 7f177e927700 1 -- 172.21.15.66:0/407999958 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f17780f6510 msgr2=0x7f17780f6930 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:34:23.067 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:23.066+0000 7f177e927700 1 --2- 172.21.15.66:0/407999958 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f17780f6510 0x7f17780f6930 secure :-1 s=READY pgs=407 cs=0 l=1 rev1=1 rx=0x7f1768009fb0 tx=0x7f1768009b30).stop 2023-11-17T05:34:23.067 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:23.066+0000 7f177e927700 1 -- 172.21.15.66:0/407999958 shutdown_connections 2023-11-17T05:34:23.067 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:23.066+0000 7f177e927700 1 --2- 172.21.15.66:0/407999958 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f17780f6510 0x7f17780f6930 unknown :-1 s=CLOSED pgs=407 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:34:23.068 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:23.066+0000 7f177e927700 1 -- 172.21.15.66:0/407999958 >> 172.21.15.66:0/407999958 conn(0x7f17780f1c70 msgr2=0x7f17780f4090 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:34:23.068 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:23.066+0000 7f177e927700 1 -- 172.21.15.66:0/407999958 shutdown_connections 2023-11-17T05:34:23.068 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:23.066+0000 7f177e927700 1 -- 172.21.15.66:0/407999958 wait complete. 2023-11-17T05:34:23.068 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:23.067+0000 7f177e927700 1 Processor -- start 2023-11-17T05:34:23.068 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:23.067+0000 7f177e927700 1 -- start start 2023-11-17T05:34:23.069 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:23.067+0000 7f177e927700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f17780f6510 0x7f17781066d0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:34:23.069 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:23.067+0000 7f177e927700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f1778106bc0 con 0x7f17780f6510 2023-11-17T05:34:23.069 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:23.068+0000 7f1777fff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f17780f6510 0x7f17781066d0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:34:23.069 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:23.068+0000 7f1777fff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f17780f6510 0x7f17781066d0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.66:48604/0 (socket says 172.21.15.66:48604) 2023-11-17T05:34:23.069 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:23.068+0000 7f1777fff700 1 -- 172.21.15.66:0/4170678275 learned_addr learned my addr 172.21.15.66:0/4170678275 (peer_addr_for_me v2:172.21.15.66:0/0) 2023-11-17T05:34:23.069 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:23.068+0000 7f1777fff700 1 -- 172.21.15.66:0/4170678275 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f1768005040 con 0x7f17780f6510 2023-11-17T05:34:23.069 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:23.068+0000 7f1777fff700 1 --2- 172.21.15.66:0/4170678275 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f17780f6510 0x7f17781066d0 secure :-1 s=READY pgs=408 cs=0 l=1 rev1=1 rx=0x7f1768013510 tx=0x7f176800b040).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:34:23.069 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:23.068+0000 7f1774ff9700 1 -- 172.21.15.66:0/4170678275 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f1768009320 con 0x7f17780f6510 2023-11-17T05:34:23.069 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:23.068+0000 7f177e927700 1 -- 172.21.15.66:0/4170678275 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f1778106dc0 con 0x7f17780f6510 2023-11-17T05:34:23.069 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:23.068+0000 7f1774ff9700 1 -- 172.21.15.66:0/4170678275 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(25 keys) v1 ==== 1074+0+0 (secure 0 0 0) 0x7f1768013f00 con 0x7f17780f6510 2023-11-17T05:34:23.070 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:23.069+0000 7f1774ff9700 1 -- 172.21.15.66:0/4170678275 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f1768014d90 con 0x7f17780f6510 2023-11-17T05:34:23.070 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:23.069+0000 7f177e927700 1 -- 172.21.15.66:0/4170678275 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f1778107250 con 0x7f17780f6510 2023-11-17T05:34:23.070 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:23.069+0000 7f1774ff9700 1 -- 172.21.15.66:0/4170678275 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 17) v1 ==== 36269+0+0 (secure 0 0 0) 0x7f17680263a0 con 0x7f17780f6510 2023-11-17T05:34:23.071 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:23.070+0000 7f1774ff9700 1 --2- 172.21.15.66:0/4170678275 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f176002ff60 0x7f1760032420 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:34:23.071 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:23.070+0000 7f177e927700 1 -- 172.21.15.66:0/4170678275 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f1778045ad0 con 0x7f17780f6510 2023-11-17T05:34:23.071 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:23.070+0000 7f1774ff9700 1 -- 172.21.15.66:0/4170678275 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(5..5 src has 1..5) v4 ==== 1270+0+0 (secure 0 0 0) 0x7f1768041100 con 0x7f17780f6510 2023-11-17T05:34:23.071 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:23.070+0000 7f17777fe700 1 --2- 172.21.15.66:0/4170678275 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f176002ff60 0x7f1760032420 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:34:23.071 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:23.070+0000 7f17777fe700 1 --2- 172.21.15.66:0/4170678275 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f176002ff60 0x7f1760032420 secure :-1 s=READY pgs=169 cs=0 l=1 rev1=1 rx=0x7f176c0096a0 tx=0x7f176c006b40).ready entity=mgr.14164 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:34:23.075 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:23.074+0000 7f1774ff9700 1 -- 172.21.15.66:0/4170678275 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+152212 (secure 0 0 0) 0x7f17680126f0 con 0x7f17780f6510 2023-11-17T05:34:23.293 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:23.292+0000 7f177e927700 1 -- 172.21.15.66:0/4170678275 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "mon dump", "format": "json"} v 0) v1 -- 0x7f1778108100 con 0x7f17780f6510 2023-11-17T05:34:23.293 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:23.293+0000 7f1774ff9700 1 -- 172.21.15.66:0/4170678275 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mon_command_ack([{"prefix": "mon dump", "format": "json"}]=0 dumped monmap epoch 1 v1) v1 ==== 95+0+690 (secure 0 0 0) 0x7f1768012330 con 0x7f17780f6510 2023-11-17T05:34:23.294 INFO:teuthology.orchestra.run.smithi066.stdout: 2023-11-17T05:34:23.294 INFO:teuthology.orchestra.run.smithi066.stdout:{"epoch":1,"fsid":"ed6470e2-8508-11ee-8dbe-212e2dc638e7","modified":"2023-11-17T05:20:13.934038Z","created":"2023-11-17T05:20:13.934038Z","min_mon_release":16,"min_mon_release_name":"pacific","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging"],"optional":[]},"mons":[{"rank":0,"name":"smithi040","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:3300","nonce":0},{"type":"v1","addr":"172.21.15.40:6789","nonce":0}]},"addr":"172.21.15.40:6789/0","public_addr":"172.21.15.40:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2023-11-17T05:34:23.295 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:23.295+0000 7f177e927700 1 -- 172.21.15.66:0/4170678275 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f176002ff60 msgr2=0x7f1760032420 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:34:23.295 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:23.295+0000 7f177e927700 1 --2- 172.21.15.66:0/4170678275 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f176002ff60 0x7f1760032420 secure :-1 s=READY pgs=169 cs=0 l=1 rev1=1 rx=0x7f176c0096a0 tx=0x7f176c006b40).stop 2023-11-17T05:34:23.295 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:23.295+0000 7f177e927700 1 -- 172.21.15.66:0/4170678275 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f17780f6510 msgr2=0x7f17781066d0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:34:23.296 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:23.295+0000 7f177e927700 1 --2- 172.21.15.66:0/4170678275 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f17780f6510 0x7f17781066d0 secure :-1 s=READY pgs=408 cs=0 l=1 rev1=1 rx=0x7f1768013510 tx=0x7f176800b040).stop 2023-11-17T05:34:23.296 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:23.295+0000 7f177e927700 1 -- 172.21.15.66:0/4170678275 shutdown_connections 2023-11-17T05:34:23.296 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:23.295+0000 7f177e927700 1 --2- 172.21.15.66:0/4170678275 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f176002ff60 0x7f1760032420 unknown :-1 s=CLOSED pgs=169 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:34:23.296 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:23.295+0000 7f177e927700 1 --2- 172.21.15.66:0/4170678275 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f17780f6510 0x7f17781066d0 unknown :-1 s=CLOSED pgs=408 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:34:23.296 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:23.295+0000 7f177e927700 1 -- 172.21.15.66:0/4170678275 >> 172.21.15.66:0/4170678275 conn(0x7f17780f1c70 msgr2=0x7f17780f2710 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:34:23.296 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:23.295+0000 7f177e927700 1 -- 172.21.15.66:0/4170678275 shutdown_connections 2023-11-17T05:34:23.296 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:23.295+0000 7f177e927700 1 -- 172.21.15.66:0/4170678275 wait complete. 2023-11-17T05:34:23.296 INFO:teuthology.orchestra.run.smithi066.stderr:dumped monmap epoch 1 2023-11-17T05:34:23.739 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:34:23 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:34:23.293822+0000 mon.smithi040 (mon.0) 320 : audit [DBG] from='client.? 172.21.15.66:0/4170678275' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-11-17T05:34:24.718 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2023-11-17T05:34:24.718 DEBUG:teuthology.orchestra.run.smithi066:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.4 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ed6470e2-8508-11ee-8dbe-212e2dc638e7 -- ceph mon dump -f json 2023-11-17T05:34:24.738 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:34:24 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:34:23.545123+0000 mgr.smithi040.ogcprl (mgr.14164) 407 : cluster [DBG] pgmap v391: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:34:25.989 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:34:25 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:34:25.545497+0000 mgr.smithi040.ogcprl (mgr.14164) 408 : cluster [DBG] pgmap v392: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:34:26.000 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:25.996+0000 7fd5df59e700 1 -- 172.21.15.66:0/3193772730 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fd5e00f51a0 msgr2=0x7fd5e00f55c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:34:26.009 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:25.996+0000 7fd5df59e700 1 --2- 172.21.15.66:0/3193772730 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fd5e00f51a0 0x7fd5e00f55c0 secure :-1 s=READY pgs=409 cs=0 l=1 rev1=1 rx=0x7fd5d0009fb0 tx=0x7fd5d0009b30).stop 2023-11-17T05:34:26.009 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:25.996+0000 7fd5df59e700 1 -- 172.21.15.66:0/3193772730 shutdown_connections 2023-11-17T05:34:26.009 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:25.996+0000 7fd5df59e700 1 --2- 172.21.15.66:0/3193772730 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fd5e00f51a0 0x7fd5e00f55c0 unknown :-1 s=CLOSED pgs=409 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:34:26.009 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:25.996+0000 7fd5df59e700 1 -- 172.21.15.66:0/3193772730 >> 172.21.15.66:0/3193772730 conn(0x7fd5e00f09a0 msgr2=0x7fd5e00f2dc0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:34:26.009 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:25.996+0000 7fd5df59e700 1 -- 172.21.15.66:0/3193772730 shutdown_connections 2023-11-17T05:34:26.009 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:25.996+0000 7fd5df59e700 1 -- 172.21.15.66:0/3193772730 wait complete. 2023-11-17T05:34:26.010 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:25.996+0000 7fd5df59e700 1 Processor -- start 2023-11-17T05:34:26.010 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:25.996+0000 7fd5df59e700 1 -- start start 2023-11-17T05:34:26.010 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:25.996+0000 7fd5df59e700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fd5e00f51a0 0x7fd5e0109170 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:34:26.010 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:25.996+0000 7fd5df59e700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fd5e0060280 con 0x7fd5e00f51a0 2023-11-17T05:34:26.010 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:25.997+0000 7fd5de59c700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fd5e00f51a0 0x7fd5e0109170 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:34:26.010 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:25.997+0000 7fd5de59c700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fd5e00f51a0 0x7fd5e0109170 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.66:48628/0 (socket says 172.21.15.66:48628) 2023-11-17T05:34:26.010 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:25.997+0000 7fd5de59c700 1 -- 172.21.15.66:0/2915643537 learned_addr learned my addr 172.21.15.66:0/2915643537 (peer_addr_for_me v2:172.21.15.66:0/0) 2023-11-17T05:34:26.010 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:25.997+0000 7fd5de59c700 1 -- 172.21.15.66:0/2915643537 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fd5d0005040 con 0x7fd5e00f51a0 2023-11-17T05:34:26.010 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:25.997+0000 7fd5de59c700 1 --2- 172.21.15.66:0/2915643537 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fd5e00f51a0 0x7fd5e0109170 secure :-1 s=READY pgs=410 cs=0 l=1 rev1=1 rx=0x7fd5d0013510 tx=0x7fd5d000b040).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:34:26.010 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:25.997+0000 7fd5ceffd700 1 -- 172.21.15.66:0/2915643537 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7fd5d0009320 con 0x7fd5e00f51a0 2023-11-17T05:34:26.010 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:25.997+0000 7fd5ceffd700 1 -- 172.21.15.66:0/2915643537 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(25 keys) v1 ==== 1074+0+0 (secure 0 0 0) 0x7fd5d0013f00 con 0x7fd5e00f51a0 2023-11-17T05:34:26.011 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:25.997+0000 7fd5ceffd700 1 -- 172.21.15.66:0/2915643537 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7fd5d0014dc0 con 0x7fd5e00f51a0 2023-11-17T05:34:26.011 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:25.997+0000 7fd5df59e700 1 -- 172.21.15.66:0/2915643537 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fd5e0060480 con 0x7fd5e00f51a0 2023-11-17T05:34:26.011 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:25.997+0000 7fd5df59e700 1 -- 172.21.15.66:0/2915643537 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fd5e01098a0 con 0x7fd5e00f51a0 2023-11-17T05:34:26.011 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:25.998+0000 7fd5ceffd700 1 -- 172.21.15.66:0/2915643537 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 17) v1 ==== 36269+0+0 (secure 0 0 0) 0x7fd5d00095f0 con 0x7fd5e00f51a0 2023-11-17T05:34:26.011 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:25.998+0000 7fd5df59e700 1 -- 172.21.15.66:0/2915643537 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fd5e0045ad0 con 0x7fd5e00f51a0 2023-11-17T05:34:26.011 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:25.998+0000 7fd5ceffd700 1 --2- 172.21.15.66:0/2915643537 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fd5c802ffb0 0x7fd5c8032470 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:34:26.011 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:25.999+0000 7fd5ceffd700 1 -- 172.21.15.66:0/2915643537 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(5..5 src has 1..5) v4 ==== 1270+0+0 (secure 0 0 0) 0x7fd5d0040ff0 con 0x7fd5e00f51a0 2023-11-17T05:34:26.011 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:25.999+0000 7fd5ddd9b700 1 --2- 172.21.15.66:0/2915643537 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fd5c802ffb0 0x7fd5c8032470 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:34:26.011 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:25.999+0000 7fd5ddd9b700 1 --2- 172.21.15.66:0/2915643537 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fd5c802ffb0 0x7fd5c8032470 secure :-1 s=READY pgs=170 cs=0 l=1 rev1=1 rx=0x7fd5d40096a0 tx=0x7fd5d4006b40).ready entity=mgr.14164 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:34:26.012 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:26.004+0000 7fd5ceffd700 1 -- 172.21.15.66:0/2915643537 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+152212 (secure 0 0 0) 0x7fd5d006d2b0 con 0x7fd5e00f51a0 2023-11-17T05:34:26.221 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:26.221+0000 7fd5df59e700 1 -- 172.21.15.66:0/2915643537 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "mon dump", "format": "json"} v 0) v1 -- 0x7fd5e00fabe0 con 0x7fd5e00f51a0 2023-11-17T05:34:26.222 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:26.222+0000 7fd5ceffd700 1 -- 172.21.15.66:0/2915643537 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mon_command_ack([{"prefix": "mon dump", "format": "json"}]=0 dumped monmap epoch 1 v1) v1 ==== 95+0+690 (secure 0 0 0) 0x7fd5d0014420 con 0x7fd5e00f51a0 2023-11-17T05:34:26.222 INFO:teuthology.orchestra.run.smithi066.stdout: 2023-11-17T05:34:26.222 INFO:teuthology.orchestra.run.smithi066.stdout:{"epoch":1,"fsid":"ed6470e2-8508-11ee-8dbe-212e2dc638e7","modified":"2023-11-17T05:20:13.934038Z","created":"2023-11-17T05:20:13.934038Z","min_mon_release":16,"min_mon_release_name":"pacific","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging"],"optional":[]},"mons":[{"rank":0,"name":"smithi040","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:3300","nonce":0},{"type":"v1","addr":"172.21.15.40:6789","nonce":0}]},"addr":"172.21.15.40:6789/0","public_addr":"172.21.15.40:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2023-11-17T05:34:26.223 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:26.223+0000 7fd5df59e700 1 -- 172.21.15.66:0/2915643537 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fd5c802ffb0 msgr2=0x7fd5c8032470 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:34:26.224 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:26.223+0000 7fd5df59e700 1 --2- 172.21.15.66:0/2915643537 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fd5c802ffb0 0x7fd5c8032470 secure :-1 s=READY pgs=170 cs=0 l=1 rev1=1 rx=0x7fd5d40096a0 tx=0x7fd5d4006b40).stop 2023-11-17T05:34:26.224 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:26.223+0000 7fd5df59e700 1 -- 172.21.15.66:0/2915643537 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fd5e00f51a0 msgr2=0x7fd5e0109170 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:34:26.224 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:26.223+0000 7fd5df59e700 1 --2- 172.21.15.66:0/2915643537 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fd5e00f51a0 0x7fd5e0109170 secure :-1 s=READY pgs=410 cs=0 l=1 rev1=1 rx=0x7fd5d0013510 tx=0x7fd5d000b040).stop 2023-11-17T05:34:26.224 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:26.223+0000 7fd5df59e700 1 -- 172.21.15.66:0/2915643537 shutdown_connections 2023-11-17T05:34:26.224 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:26.223+0000 7fd5df59e700 1 --2- 172.21.15.66:0/2915643537 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fd5c802ffb0 0x7fd5c8032470 unknown :-1 s=CLOSED pgs=170 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:34:26.224 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:26.223+0000 7fd5df59e700 1 --2- 172.21.15.66:0/2915643537 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fd5e00f51a0 0x7fd5e0109170 unknown :-1 s=CLOSED pgs=410 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:34:26.224 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:26.223+0000 7fd5df59e700 1 -- 172.21.15.66:0/2915643537 >> 172.21.15.66:0/2915643537 conn(0x7fd5e00f09a0 msgr2=0x7fd5e00f1460 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:34:26.224 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:26.223+0000 7fd5df59e700 1 -- 172.21.15.66:0/2915643537 shutdown_connections 2023-11-17T05:34:26.224 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:26.223+0000 7fd5df59e700 1 -- 172.21.15.66:0/2915643537 wait complete. 2023-11-17T05:34:26.224 INFO:teuthology.orchestra.run.smithi066.stderr:dumped monmap epoch 1 2023-11-17T05:34:26.989 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:34:26 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:34:26.222516+0000 mon.smithi040 (mon.0) 321 : audit [DBG] from='client.? 172.21.15.66:0/2915643537' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-11-17T05:34:27.675 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2023-11-17T05:34:27.676 DEBUG:teuthology.orchestra.run.smithi066:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.4 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ed6470e2-8508-11ee-8dbe-212e2dc638e7 -- ceph mon dump -f json 2023-11-17T05:34:27.988 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:34:27 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:34:27.545863+0000 mgr.smithi040.ogcprl (mgr.14164) 409 : cluster [DBG] pgmap v393: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:34:28.951 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:28.950+0000 7fb75d01e700 1 -- 172.21.15.66:0/191658635 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fb7580f62a0 msgr2=0x7fb7580f66c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:34:28.952 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:28.950+0000 7fb75d01e700 1 --2- 172.21.15.66:0/191658635 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fb7580f62a0 0x7fb7580f66c0 secure :-1 s=READY pgs=411 cs=0 l=1 rev1=1 rx=0x7fb748009fb0 tx=0x7fb748009b30).stop 2023-11-17T05:34:28.952 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:28.951+0000 7fb75d01e700 1 -- 172.21.15.66:0/191658635 shutdown_connections 2023-11-17T05:34:28.952 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:28.951+0000 7fb75d01e700 1 --2- 172.21.15.66:0/191658635 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fb7580f62a0 0x7fb7580f66c0 unknown :-1 s=CLOSED pgs=411 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:34:28.952 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:28.951+0000 7fb75d01e700 1 -- 172.21.15.66:0/191658635 >> 172.21.15.66:0/191658635 conn(0x7fb7580f1aa0 msgr2=0x7fb7580f3ec0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:34:28.952 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:28.951+0000 7fb75d01e700 1 -- 172.21.15.66:0/191658635 shutdown_connections 2023-11-17T05:34:28.952 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:28.951+0000 7fb75d01e700 1 -- 172.21.15.66:0/191658635 wait complete. 2023-11-17T05:34:28.952 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:28.951+0000 7fb75d01e700 1 Processor -- start 2023-11-17T05:34:28.952 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:28.951+0000 7fb75d01e700 1 -- start start 2023-11-17T05:34:28.952 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:28.951+0000 7fb75d01e700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fb7580f62a0 0x7fb75806b9a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:34:28.952 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:28.951+0000 7fb75d01e700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fb758069e20 con 0x7fb7580f62a0 2023-11-17T05:34:28.955 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:28.952+0000 7fb757fff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fb7580f62a0 0x7fb75806b9a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:34:28.955 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:28.952+0000 7fb757fff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fb7580f62a0 0x7fb75806b9a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.66:48644/0 (socket says 172.21.15.66:48644) 2023-11-17T05:34:28.955 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:28.952+0000 7fb757fff700 1 -- 172.21.15.66:0/3533683444 learned_addr learned my addr 172.21.15.66:0/3533683444 (peer_addr_for_me v2:172.21.15.66:0/0) 2023-11-17T05:34:28.955 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:28.952+0000 7fb757fff700 1 -- 172.21.15.66:0/3533683444 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fb748005040 con 0x7fb7580f62a0 2023-11-17T05:34:28.955 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:28.952+0000 7fb757fff700 1 --2- 172.21.15.66:0/3533683444 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fb7580f62a0 0x7fb75806b9a0 secure :-1 s=READY pgs=412 cs=0 l=1 rev1=1 rx=0x7fb748013510 tx=0x7fb74800b040).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:34:28.955 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:28.952+0000 7fb754ff9700 1 -- 172.21.15.66:0/3533683444 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7fb748009320 con 0x7fb7580f62a0 2023-11-17T05:34:28.955 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:28.952+0000 7fb75d01e700 1 -- 172.21.15.66:0/3533683444 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fb758069fc0 con 0x7fb7580f62a0 2023-11-17T05:34:28.955 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:28.952+0000 7fb754ff9700 1 -- 172.21.15.66:0/3533683444 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(25 keys) v1 ==== 1074+0+0 (secure 0 0 0) 0x7fb748013f00 con 0x7fb7580f62a0 2023-11-17T05:34:28.955 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:28.952+0000 7fb754ff9700 1 -- 172.21.15.66:0/3533683444 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7fb748014dc0 con 0x7fb7580f62a0 2023-11-17T05:34:28.955 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:28.952+0000 7fb75d01e700 1 -- 172.21.15.66:0/3533683444 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fb75806a450 con 0x7fb7580f62a0 2023-11-17T05:34:28.956 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:28.953+0000 7fb754ff9700 1 -- 172.21.15.66:0/3533683444 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 17) v1 ==== 36269+0+0 (secure 0 0 0) 0x7fb7480095f0 con 0x7fb7580f62a0 2023-11-17T05:34:28.956 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:28.953+0000 7fb75d01e700 1 -- 172.21.15.66:0/3533683444 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fb758059100 con 0x7fb7580f62a0 2023-11-17T05:34:28.956 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:28.953+0000 7fb754ff9700 1 --2- 172.21.15.66:0/3533683444 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fb74002ff60 0x7fb740032420 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:34:28.956 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:28.953+0000 7fb754ff9700 1 -- 172.21.15.66:0/3533683444 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(5..5 src has 1..5) v4 ==== 1270+0+0 (secure 0 0 0) 0x7fb7480410e0 con 0x7fb7580f62a0 2023-11-17T05:34:28.956 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:28.954+0000 7fb7577fe700 1 --2- 172.21.15.66:0/3533683444 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fb74002ff60 0x7fb740032420 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:34:28.956 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:28.954+0000 7fb7577fe700 1 --2- 172.21.15.66:0/3533683444 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fb74002ff60 0x7fb740032420 secure :-1 s=READY pgs=171 cs=0 l=1 rev1=1 rx=0x7fb74c0096a0 tx=0x7fb74c006b40).ready entity=mgr.14164 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:34:28.958 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:28.958+0000 7fb754ff9700 1 -- 172.21.15.66:0/3533683444 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+152212 (secure 0 0 0) 0x7fb748014420 con 0x7fb7580f62a0 2023-11-17T05:34:29.175 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:29.175+0000 7fb75d01e700 1 -- 172.21.15.66:0/3533683444 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "mon dump", "format": "json"} v 0) v1 -- 0x7fb758045ad0 con 0x7fb7580f62a0 2023-11-17T05:34:29.176 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:29.176+0000 7fb754ff9700 1 -- 172.21.15.66:0/3533683444 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mon_command_ack([{"prefix": "mon dump", "format": "json"}]=0 dumped monmap epoch 1 v1) v1 ==== 95+0+690 (secure 0 0 0) 0x7fb74801c920 con 0x7fb7580f62a0 2023-11-17T05:34:29.177 INFO:teuthology.orchestra.run.smithi066.stdout: 2023-11-17T05:34:29.177 INFO:teuthology.orchestra.run.smithi066.stdout:{"epoch":1,"fsid":"ed6470e2-8508-11ee-8dbe-212e2dc638e7","modified":"2023-11-17T05:20:13.934038Z","created":"2023-11-17T05:20:13.934038Z","min_mon_release":16,"min_mon_release_name":"pacific","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging"],"optional":[]},"mons":[{"rank":0,"name":"smithi040","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:3300","nonce":0},{"type":"v1","addr":"172.21.15.40:6789","nonce":0}]},"addr":"172.21.15.40:6789/0","public_addr":"172.21.15.40:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2023-11-17T05:34:29.178 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:29.178+0000 7fb75d01e700 1 -- 172.21.15.66:0/3533683444 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fb74002ff60 msgr2=0x7fb740032420 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:34:29.178 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:29.178+0000 7fb75d01e700 1 --2- 172.21.15.66:0/3533683444 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fb74002ff60 0x7fb740032420 secure :-1 s=READY pgs=171 cs=0 l=1 rev1=1 rx=0x7fb74c0096a0 tx=0x7fb74c006b40).stop 2023-11-17T05:34:29.178 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:29.178+0000 7fb75d01e700 1 -- 172.21.15.66:0/3533683444 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fb7580f62a0 msgr2=0x7fb75806b9a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:34:29.178 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:29.178+0000 7fb75d01e700 1 --2- 172.21.15.66:0/3533683444 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fb7580f62a0 0x7fb75806b9a0 secure :-1 s=READY pgs=412 cs=0 l=1 rev1=1 rx=0x7fb748013510 tx=0x7fb74800b040).stop 2023-11-17T05:34:29.179 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:29.178+0000 7fb75d01e700 1 -- 172.21.15.66:0/3533683444 shutdown_connections 2023-11-17T05:34:29.179 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:29.178+0000 7fb75d01e700 1 --2- 172.21.15.66:0/3533683444 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fb74002ff60 0x7fb740032420 unknown :-1 s=CLOSED pgs=171 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:34:29.179 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:29.178+0000 7fb75d01e700 1 --2- 172.21.15.66:0/3533683444 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fb7580f62a0 0x7fb75806b9a0 unknown :-1 s=CLOSED pgs=412 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:34:29.179 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:29.178+0000 7fb75d01e700 1 -- 172.21.15.66:0/3533683444 >> 172.21.15.66:0/3533683444 conn(0x7fb7580f1aa0 msgr2=0x7fb7580f2510 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:34:29.179 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:29.178+0000 7fb75d01e700 1 -- 172.21.15.66:0/3533683444 shutdown_connections 2023-11-17T05:34:29.179 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:29.178+0000 7fb75d01e700 1 -- 172.21.15.66:0/3533683444 wait complete. 2023-11-17T05:34:29.179 INFO:teuthology.orchestra.run.smithi066.stderr:dumped monmap epoch 1 2023-11-17T05:34:29.489 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:34:29 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:34:29.176705+0000 mon.smithi040 (mon.0) 322 : audit [DBG] from='client.? 172.21.15.66:0/3533683444' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-11-17T05:34:30.559 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2023-11-17T05:34:30.560 DEBUG:teuthology.orchestra.run.smithi066:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.4 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ed6470e2-8508-11ee-8dbe-212e2dc638e7 -- ceph mon dump -f json 2023-11-17T05:34:30.988 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:34:30 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:34:29.546205+0000 mgr.smithi040.ogcprl (mgr.14164) 410 : cluster [DBG] pgmap v394: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:34:31.720 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:31.715+0000 7f964882c700 1 -- 172.21.15.66:0/1310527795 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f96440f51a0 msgr2=0x7f96440f55c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:34:31.721 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:31.715+0000 7f964882c700 1 --2- 172.21.15.66:0/1310527795 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f96440f51a0 0x7f96440f55c0 secure :-1 s=READY pgs=413 cs=0 l=1 rev1=1 rx=0x7f9634009fb0 tx=0x7f9634009b30).stop 2023-11-17T05:34:31.721 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:31.715+0000 7f964882c700 1 -- 172.21.15.66:0/1310527795 shutdown_connections 2023-11-17T05:34:31.722 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:31.715+0000 7f964882c700 1 --2- 172.21.15.66:0/1310527795 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f96440f51a0 0x7f96440f55c0 unknown :-1 s=CLOSED pgs=413 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:34:31.722 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:31.715+0000 7f964882c700 1 -- 172.21.15.66:0/1310527795 >> 172.21.15.66:0/1310527795 conn(0x7f96440f09a0 msgr2=0x7f96440f2dc0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:34:31.722 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:31.715+0000 7f964882c700 1 -- 172.21.15.66:0/1310527795 shutdown_connections 2023-11-17T05:34:31.723 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:31.715+0000 7f964882c700 1 -- 172.21.15.66:0/1310527795 wait complete. 2023-11-17T05:34:31.723 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:31.715+0000 7f964882c700 1 Processor -- start 2023-11-17T05:34:31.723 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:31.716+0000 7f964882c700 1 -- start start 2023-11-17T05:34:31.723 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:31.716+0000 7f964882c700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f96440f51a0 0x7f96441053b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:34:31.723 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:31.716+0000 7f964882c700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f96441058a0 con 0x7f96440f51a0 2023-11-17T05:34:31.723 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:31.716+0000 7f96437fe700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f96440f51a0 0x7f96441053b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:34:31.728 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:31.716+0000 7f96437fe700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f96440f51a0 0x7f96441053b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.66:48662/0 (socket says 172.21.15.66:48662) 2023-11-17T05:34:31.728 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:31.716+0000 7f96437fe700 1 -- 172.21.15.66:0/446777416 learned_addr learned my addr 172.21.15.66:0/446777416 (peer_addr_for_me v2:172.21.15.66:0/0) 2023-11-17T05:34:31.728 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:31.716+0000 7f96437fe700 1 -- 172.21.15.66:0/446777416 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f9634005040 con 0x7f96440f51a0 2023-11-17T05:34:31.728 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:31.716+0000 7f96437fe700 1 --2- 172.21.15.66:0/446777416 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f96440f51a0 0x7f96441053b0 secure :-1 s=READY pgs=414 cs=0 l=1 rev1=1 rx=0x7f9634013510 tx=0x7f963400b040).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:34:31.728 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:31.717+0000 7f962bfff700 1 -- 172.21.15.66:0/446777416 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f9634009320 con 0x7f96440f51a0 2023-11-17T05:34:31.728 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:31.717+0000 7f962bfff700 1 -- 172.21.15.66:0/446777416 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(25 keys) v1 ==== 1074+0+0 (secure 0 0 0) 0x7f9634013f00 con 0x7f96440f51a0 2023-11-17T05:34:31.728 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:31.717+0000 7f964882c700 1 -- 172.21.15.66:0/446777416 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f9644105aa0 con 0x7f96440f51a0 2023-11-17T05:34:31.728 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:31.717+0000 7f962bfff700 1 -- 172.21.15.66:0/446777416 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f9634014dc0 con 0x7f96440f51a0 2023-11-17T05:34:31.729 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:31.717+0000 7f964882c700 1 -- 172.21.15.66:0/446777416 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f9644105f30 con 0x7f96440f51a0 2023-11-17T05:34:31.729 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:31.718+0000 7f962bfff700 1 -- 172.21.15.66:0/446777416 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 17) v1 ==== 36269+0+0 (secure 0 0 0) 0x7f96340095f0 con 0x7f96440f51a0 2023-11-17T05:34:31.729 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:31.718+0000 7f964882c700 1 -- 172.21.15.66:0/446777416 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f9644045ad0 con 0x7f96440f51a0 2023-11-17T05:34:31.729 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:31.718+0000 7f962bfff700 1 --2- 172.21.15.66:0/446777416 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f962c02ff60 0x7f962c032420 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:34:31.729 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:31.718+0000 7f962bfff700 1 -- 172.21.15.66:0/446777416 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(5..5 src has 1..5) v4 ==== 1270+0+0 (secure 0 0 0) 0x7f9634041040 con 0x7f96440f51a0 2023-11-17T05:34:31.729 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:31.718+0000 7f9642ffd700 1 --2- 172.21.15.66:0/446777416 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f962c02ff60 0x7f962c032420 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:34:31.729 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:31.718+0000 7f9642ffd700 1 --2- 172.21.15.66:0/446777416 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f962c02ff60 0x7f962c032420 secure :-1 s=READY pgs=172 cs=0 l=1 rev1=1 rx=0x7f96380096a0 tx=0x7f9638006b40).ready entity=mgr.14164 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:34:31.729 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:31.723+0000 7f962bfff700 1 -- 172.21.15.66:0/446777416 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+152212 (secure 0 0 0) 0x7f9634014420 con 0x7f96440f51a0 2023-11-17T05:34:31.943 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:31.943+0000 7f964882c700 1 -- 172.21.15.66:0/446777416 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "mon dump", "format": "json"} v 0) v1 -- 0x7f9644106de0 con 0x7f96440f51a0 2023-11-17T05:34:31.944 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:31.944+0000 7f962bfff700 1 -- 172.21.15.66:0/446777416 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mon_command_ack([{"prefix": "mon dump", "format": "json"}]=0 dumped monmap epoch 1 v1) v1 ==== 95+0+690 (secure 0 0 0) 0x7f9634010230 con 0x7f96440f51a0 2023-11-17T05:34:31.945 INFO:teuthology.orchestra.run.smithi066.stdout: 2023-11-17T05:34:31.945 INFO:teuthology.orchestra.run.smithi066.stdout:{"epoch":1,"fsid":"ed6470e2-8508-11ee-8dbe-212e2dc638e7","modified":"2023-11-17T05:20:13.934038Z","created":"2023-11-17T05:20:13.934038Z","min_mon_release":16,"min_mon_release_name":"pacific","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging"],"optional":[]},"mons":[{"rank":0,"name":"smithi040","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:3300","nonce":0},{"type":"v1","addr":"172.21.15.40:6789","nonce":0}]},"addr":"172.21.15.40:6789/0","public_addr":"172.21.15.40:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2023-11-17T05:34:31.947 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:31.945+0000 7f964882c700 1 -- 172.21.15.66:0/446777416 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f962c02ff60 msgr2=0x7f962c032420 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:34:31.948 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:31.945+0000 7f964882c700 1 --2- 172.21.15.66:0/446777416 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f962c02ff60 0x7f962c032420 secure :-1 s=READY pgs=172 cs=0 l=1 rev1=1 rx=0x7f96380096a0 tx=0x7f9638006b40).stop 2023-11-17T05:34:31.948 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:31.945+0000 7f964882c700 1 -- 172.21.15.66:0/446777416 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f96440f51a0 msgr2=0x7f96441053b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:34:31.948 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:31.945+0000 7f964882c700 1 --2- 172.21.15.66:0/446777416 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f96440f51a0 0x7f96441053b0 secure :-1 s=READY pgs=414 cs=0 l=1 rev1=1 rx=0x7f9634013510 tx=0x7f963400b040).stop 2023-11-17T05:34:31.948 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:31.945+0000 7f964882c700 1 -- 172.21.15.66:0/446777416 shutdown_connections 2023-11-17T05:34:31.949 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:31.945+0000 7f964882c700 1 --2- 172.21.15.66:0/446777416 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f962c02ff60 0x7f962c032420 unknown :-1 s=CLOSED pgs=172 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:34:31.949 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:31.945+0000 7f964882c700 1 --2- 172.21.15.66:0/446777416 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f96440f51a0 0x7f96441053b0 unknown :-1 s=CLOSED pgs=414 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:34:31.949 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:31.945+0000 7f964882c700 1 -- 172.21.15.66:0/446777416 >> 172.21.15.66:0/446777416 conn(0x7f96440f09a0 msgr2=0x7f96440f1460 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:34:31.949 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:31.945+0000 7f964882c700 1 -- 172.21.15.66:0/446777416 shutdown_connections 2023-11-17T05:34:31.950 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:31.945+0000 7f964882c700 1 -- 172.21.15.66:0/446777416 wait complete. 2023-11-17T05:34:31.950 INFO:teuthology.orchestra.run.smithi066.stderr:dumped monmap epoch 1 2023-11-17T05:34:31.989 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:34:31 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:34:31.546485+0000 mgr.smithi040.ogcprl (mgr.14164) 411 : cluster [DBG] pgmap v395: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:34:32.989 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:34:32 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:34:31.944128+0000 mon.smithi040 (mon.0) 323 : audit [DBG] from='client.? 172.21.15.66:0/446777416' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-11-17T05:34:33.325 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2023-11-17T05:34:33.325 DEBUG:teuthology.orchestra.run.smithi066:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.4 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ed6470e2-8508-11ee-8dbe-212e2dc638e7 -- ceph mon dump -f json 2023-11-17T05:34:33.989 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:34:33 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:34:33.546874+0000 mgr.smithi040.ogcprl (mgr.14164) 412 : cluster [DBG] pgmap v396: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:34:34.499 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:34.497+0000 7f14a9a76700 1 -- 172.21.15.66:0/2657668340 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f14a40f4b00 msgr2=0x7f14a40f4f20 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:34:34.499 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:34.497+0000 7f14a9a76700 1 --2- 172.21.15.66:0/2657668340 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f14a40f4b00 0x7f14a40f4f20 secure :-1 s=READY pgs=415 cs=0 l=1 rev1=1 rx=0x7f1494009fb0 tx=0x7f1494009b30).stop 2023-11-17T05:34:34.499 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:34.497+0000 7f14a9a76700 1 -- 172.21.15.66:0/2657668340 shutdown_connections 2023-11-17T05:34:34.499 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:34.497+0000 7f14a9a76700 1 --2- 172.21.15.66:0/2657668340 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f14a40f4b00 0x7f14a40f4f20 unknown :-1 s=CLOSED pgs=415 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:34:34.499 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:34.497+0000 7f14a9a76700 1 -- 172.21.15.66:0/2657668340 >> 172.21.15.66:0/2657668340 conn(0x7f14a40f0260 msgr2=0x7f14a40f2680 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:34:34.499 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:34.497+0000 7f14a9a76700 1 -- 172.21.15.66:0/2657668340 shutdown_connections 2023-11-17T05:34:34.499 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:34.497+0000 7f14a9a76700 1 -- 172.21.15.66:0/2657668340 wait complete. 2023-11-17T05:34:34.499 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:34.497+0000 7f14a9a76700 1 Processor -- start 2023-11-17T05:34:34.500 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:34.498+0000 7f14a9a76700 1 -- start start 2023-11-17T05:34:34.500 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:34.498+0000 7f14a9a76700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f14a40f4b00 0x7f14a4106710 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:34:34.500 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:34.498+0000 7f14a9a76700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f14a4106c00 con 0x7f14a40f4b00 2023-11-17T05:34:34.500 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:34.498+0000 7f14a37fe700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f14a40f4b00 0x7f14a4106710 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:34:34.500 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:34.498+0000 7f14a37fe700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f14a40f4b00 0x7f14a4106710 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.66:33134/0 (socket says 172.21.15.66:33134) 2023-11-17T05:34:34.500 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:34.498+0000 7f14a37fe700 1 -- 172.21.15.66:0/564750074 learned_addr learned my addr 172.21.15.66:0/564750074 (peer_addr_for_me v2:172.21.15.66:0/0) 2023-11-17T05:34:34.500 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:34.498+0000 7f14a37fe700 1 -- 172.21.15.66:0/564750074 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f1494005040 con 0x7f14a40f4b00 2023-11-17T05:34:34.500 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:34.498+0000 7f14a37fe700 1 --2- 172.21.15.66:0/564750074 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f14a40f4b00 0x7f14a4106710 secure :-1 s=READY pgs=416 cs=0 l=1 rev1=1 rx=0x7f1494013510 tx=0x7f149400b040).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:34:34.500 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:34.499+0000 7f149bfff700 1 -- 172.21.15.66:0/564750074 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f1494009320 con 0x7f14a40f4b00 2023-11-17T05:34:34.500 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:34.499+0000 7f149bfff700 1 -- 172.21.15.66:0/564750074 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(25 keys) v1 ==== 1074+0+0 (secure 0 0 0) 0x7f1494013f00 con 0x7f14a40f4b00 2023-11-17T05:34:34.500 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:34.499+0000 7f14a9a76700 1 -- 172.21.15.66:0/564750074 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f14a4106e00 con 0x7f14a40f4b00 2023-11-17T05:34:34.500 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:34.499+0000 7f149bfff700 1 -- 172.21.15.66:0/564750074 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f1494014d90 con 0x7f14a40f4b00 2023-11-17T05:34:34.501 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:34.499+0000 7f14a9a76700 1 -- 172.21.15.66:0/564750074 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f14a4107290 con 0x7f14a40f4b00 2023-11-17T05:34:34.501 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:34.499+0000 7f149bfff700 1 -- 172.21.15.66:0/564750074 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 17) v1 ==== 36269+0+0 (secure 0 0 0) 0x7f14940095f0 con 0x7f14a40f4b00 2023-11-17T05:34:34.502 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:34.500+0000 7f14a9a76700 1 -- 172.21.15.66:0/564750074 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f14a4045ad0 con 0x7f14a40f4b00 2023-11-17T05:34:34.502 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:34.500+0000 7f149bfff700 1 --2- 172.21.15.66:0/564750074 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f149002ff60 0x7f1490032420 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:34:34.502 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:34.500+0000 7f149bfff700 1 -- 172.21.15.66:0/564750074 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(5..5 src has 1..5) v4 ==== 1270+0+0 (secure 0 0 0) 0x7f1494040fe0 con 0x7f14a40f4b00 2023-11-17T05:34:34.502 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:34.500+0000 7f14a2ffd700 1 --2- 172.21.15.66:0/564750074 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f149002ff60 0x7f1490032420 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:34:34.502 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:34.500+0000 7f14a2ffd700 1 --2- 172.21.15.66:0/564750074 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f149002ff60 0x7f1490032420 secure :-1 s=READY pgs=173 cs=0 l=1 rev1=1 rx=0x7f148c0096a0 tx=0x7f148c006b40).ready entity=mgr.14164 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:34:34.506 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:34.505+0000 7f149bfff700 1 -- 172.21.15.66:0/564750074 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+152212 (secure 0 0 0) 0x7f149401d7f0 con 0x7f14a40f4b00 2023-11-17T05:34:34.724 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:34.724+0000 7f14a9a76700 1 -- 172.21.15.66:0/564750074 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "mon dump", "format": "json"} v 0) v1 -- 0x7f14a40fa5c0 con 0x7f14a40f4b00 2023-11-17T05:34:34.725 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:34.724+0000 7f149bfff700 1 -- 172.21.15.66:0/564750074 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mon_command_ack([{"prefix": "mon dump", "format": "json"}]=0 dumped monmap epoch 1 v1) v1 ==== 95+0+690 (secure 0 0 0) 0x7f1494010230 con 0x7f14a40f4b00 2023-11-17T05:34:34.725 INFO:teuthology.orchestra.run.smithi066.stdout: 2023-11-17T05:34:34.725 INFO:teuthology.orchestra.run.smithi066.stdout:{"epoch":1,"fsid":"ed6470e2-8508-11ee-8dbe-212e2dc638e7","modified":"2023-11-17T05:20:13.934038Z","created":"2023-11-17T05:20:13.934038Z","min_mon_release":16,"min_mon_release_name":"pacific","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging"],"optional":[]},"mons":[{"rank":0,"name":"smithi040","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:3300","nonce":0},{"type":"v1","addr":"172.21.15.40:6789","nonce":0}]},"addr":"172.21.15.40:6789/0","public_addr":"172.21.15.40:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2023-11-17T05:34:34.726 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:34.726+0000 7f14a9a76700 1 -- 172.21.15.66:0/564750074 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f149002ff60 msgr2=0x7f1490032420 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:34:34.726 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:34.726+0000 7f14a9a76700 1 --2- 172.21.15.66:0/564750074 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f149002ff60 0x7f1490032420 secure :-1 s=READY pgs=173 cs=0 l=1 rev1=1 rx=0x7f148c0096a0 tx=0x7f148c006b40).stop 2023-11-17T05:34:34.726 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:34.726+0000 7f14a9a76700 1 -- 172.21.15.66:0/564750074 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f14a40f4b00 msgr2=0x7f14a4106710 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:34:34.726 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:34.726+0000 7f14a9a76700 1 --2- 172.21.15.66:0/564750074 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f14a40f4b00 0x7f14a4106710 secure :-1 s=READY pgs=416 cs=0 l=1 rev1=1 rx=0x7f1494013510 tx=0x7f149400b040).stop 2023-11-17T05:34:34.727 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:34.726+0000 7f14a9a76700 1 -- 172.21.15.66:0/564750074 shutdown_connections 2023-11-17T05:34:34.727 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:34.726+0000 7f14a9a76700 1 --2- 172.21.15.66:0/564750074 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f149002ff60 0x7f1490032420 unknown :-1 s=CLOSED pgs=173 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:34:34.727 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:34.726+0000 7f14a9a76700 1 --2- 172.21.15.66:0/564750074 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f14a40f4b00 0x7f14a4106710 unknown :-1 s=CLOSED pgs=416 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:34:34.727 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:34.726+0000 7f14a9a76700 1 -- 172.21.15.66:0/564750074 >> 172.21.15.66:0/564750074 conn(0x7f14a40f0260 msgr2=0x7f14a40f25b0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:34:34.727 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:34.726+0000 7f14a9a76700 1 -- 172.21.15.66:0/564750074 shutdown_connections 2023-11-17T05:34:34.727 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:34.726+0000 7f14a9a76700 1 -- 172.21.15.66:0/564750074 wait complete. 2023-11-17T05:34:34.727 INFO:teuthology.orchestra.run.smithi066.stderr:dumped monmap epoch 1 2023-11-17T05:34:35.988 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:34:35 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:34:34.725008+0000 mon.smithi040 (mon.0) 324 : audit [DBG] from='client.? 172.21.15.66:0/564750074' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-11-17T05:34:36.113 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2023-11-17T05:34:36.114 DEBUG:teuthology.orchestra.run.smithi066:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.4 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ed6470e2-8508-11ee-8dbe-212e2dc638e7 -- ceph mon dump -f json 2023-11-17T05:34:36.989 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:34:36 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:34:35.547287+0000 mgr.smithi040.ogcprl (mgr.14164) 413 : cluster [DBG] pgmap v397: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:34:37.327 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:37.326+0000 7f7749d24700 1 -- 172.21.15.66:0/2185279495 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f77440f51a0 msgr2=0x7f77440f55c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:34:37.327 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:37.326+0000 7f7749d24700 1 --2- 172.21.15.66:0/2185279495 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f77440f51a0 0x7f77440f55c0 secure :-1 s=READY pgs=417 cs=0 l=1 rev1=1 rx=0x7f7738009fb0 tx=0x7f7738009b30).stop 2023-11-17T05:34:37.327 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:37.326+0000 7f7749d24700 1 -- 172.21.15.66:0/2185279495 shutdown_connections 2023-11-17T05:34:37.327 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:37.326+0000 7f7749d24700 1 --2- 172.21.15.66:0/2185279495 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f77440f51a0 0x7f77440f55c0 unknown :-1 s=CLOSED pgs=417 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:34:37.327 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:37.326+0000 7f7749d24700 1 -- 172.21.15.66:0/2185279495 >> 172.21.15.66:0/2185279495 conn(0x7f77440f09a0 msgr2=0x7f77440f2dc0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:34:37.327 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:37.326+0000 7f7749d24700 1 -- 172.21.15.66:0/2185279495 shutdown_connections 2023-11-17T05:34:37.327 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:37.326+0000 7f7749d24700 1 -- 172.21.15.66:0/2185279495 wait complete. 2023-11-17T05:34:37.327 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:37.327+0000 7f7749d24700 1 Processor -- start 2023-11-17T05:34:37.327 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:37.327+0000 7f7749d24700 1 -- start start 2023-11-17T05:34:37.328 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:37.327+0000 7f7749d24700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f77440f51a0 0x7f77441064f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:34:37.329 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:37.327+0000 7f7749d24700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f77441069e0 con 0x7f77440f51a0 2023-11-17T05:34:37.329 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:37.327+0000 7f7748d22700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f77440f51a0 0x7f77441064f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:34:37.329 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:37.327+0000 7f7748d22700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f77440f51a0 0x7f77441064f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.66:33150/0 (socket says 172.21.15.66:33150) 2023-11-17T05:34:37.329 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:37.328+0000 7f7748d22700 1 -- 172.21.15.66:0/1227490212 learned_addr learned my addr 172.21.15.66:0/1227490212 (peer_addr_for_me v2:172.21.15.66:0/0) 2023-11-17T05:34:37.329 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:37.328+0000 7f7748d22700 1 -- 172.21.15.66:0/1227490212 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f7738005040 con 0x7f77440f51a0 2023-11-17T05:34:37.329 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:37.328+0000 7f7748d22700 1 --2- 172.21.15.66:0/1227490212 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f77440f51a0 0x7f77441064f0 secure :-1 s=READY pgs=418 cs=0 l=1 rev1=1 rx=0x7f7738013510 tx=0x7f773800b040).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:34:37.329 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:37.328+0000 7f77417fa700 1 -- 172.21.15.66:0/1227490212 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f7738009320 con 0x7f77440f51a0 2023-11-17T05:34:37.329 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:37.328+0000 7f7749d24700 1 -- 172.21.15.66:0/1227490212 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f7744106be0 con 0x7f77440f51a0 2023-11-17T05:34:37.329 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:37.328+0000 7f77417fa700 1 -- 172.21.15.66:0/1227490212 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(25 keys) v1 ==== 1074+0+0 (secure 0 0 0) 0x7f7738013f00 con 0x7f77440f51a0 2023-11-17T05:34:37.329 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:37.328+0000 7f77417fa700 1 -- 172.21.15.66:0/1227490212 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f7738014dc0 con 0x7f77440f51a0 2023-11-17T05:34:37.329 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:37.328+0000 7f7749d24700 1 -- 172.21.15.66:0/1227490212 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f7744107070 con 0x7f77440f51a0 2023-11-17T05:34:37.330 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:37.329+0000 7f77417fa700 1 -- 172.21.15.66:0/1227490212 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 17) v1 ==== 36269+0+0 (secure 0 0 0) 0x7f773801d3d0 con 0x7f77440f51a0 2023-11-17T05:34:37.330 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:37.329+0000 7f77417fa700 1 --2- 172.21.15.66:0/1227490212 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f773402ff60 0x7f7734032420 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:34:37.330 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:37.329+0000 7f7749d24700 1 -- 172.21.15.66:0/1227490212 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f7744045ad0 con 0x7f77440f51a0 2023-11-17T05:34:37.330 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:37.329+0000 7f77417fa700 1 -- 172.21.15.66:0/1227490212 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(5..5 src has 1..5) v4 ==== 1270+0+0 (secure 0 0 0) 0x7f7738041320 con 0x7f77440f51a0 2023-11-17T05:34:37.331 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:37.330+0000 7f7743fff700 1 --2- 172.21.15.66:0/1227490212 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f773402ff60 0x7f7734032420 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:34:37.331 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:37.330+0000 7f7743fff700 1 --2- 172.21.15.66:0/1227490212 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f773402ff60 0x7f7734032420 secure :-1 s=READY pgs=174 cs=0 l=1 rev1=1 rx=0x7f77300096a0 tx=0x7f7730006b40).ready entity=mgr.14164 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:34:37.334 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:37.334+0000 7f77417fa700 1 -- 172.21.15.66:0/1227490212 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+152212 (secure 0 0 0) 0x7f7738026490 con 0x7f77440f51a0 2023-11-17T05:34:37.554 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:37.553+0000 7f7749d24700 1 -- 172.21.15.66:0/1227490212 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "mon dump", "format": "json"} v 0) v1 -- 0x7f7744107ec0 con 0x7f77440f51a0 2023-11-17T05:34:37.554 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:37.554+0000 7f77417fa700 1 -- 172.21.15.66:0/1227490212 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mon_command_ack([{"prefix": "mon dump", "format": "json"}]=0 dumped monmap epoch 1 v1) v1 ==== 95+0+690 (secure 0 0 0) 0x7f7738026490 con 0x7f77440f51a0 2023-11-17T05:34:37.554 INFO:teuthology.orchestra.run.smithi066.stdout: 2023-11-17T05:34:37.555 INFO:teuthology.orchestra.run.smithi066.stdout:{"epoch":1,"fsid":"ed6470e2-8508-11ee-8dbe-212e2dc638e7","modified":"2023-11-17T05:20:13.934038Z","created":"2023-11-17T05:20:13.934038Z","min_mon_release":16,"min_mon_release_name":"pacific","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging"],"optional":[]},"mons":[{"rank":0,"name":"smithi040","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:3300","nonce":0},{"type":"v1","addr":"172.21.15.40:6789","nonce":0}]},"addr":"172.21.15.40:6789/0","public_addr":"172.21.15.40:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2023-11-17T05:34:37.556 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:37.555+0000 7f7749d24700 1 -- 172.21.15.66:0/1227490212 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f773402ff60 msgr2=0x7f7734032420 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:34:37.556 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:37.555+0000 7f7749d24700 1 --2- 172.21.15.66:0/1227490212 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f773402ff60 0x7f7734032420 secure :-1 s=READY pgs=174 cs=0 l=1 rev1=1 rx=0x7f77300096a0 tx=0x7f7730006b40).stop 2023-11-17T05:34:37.556 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:37.555+0000 7f7749d24700 1 -- 172.21.15.66:0/1227490212 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f77440f51a0 msgr2=0x7f77441064f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:34:37.556 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:37.555+0000 7f7749d24700 1 --2- 172.21.15.66:0/1227490212 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f77440f51a0 0x7f77441064f0 secure :-1 s=READY pgs=418 cs=0 l=1 rev1=1 rx=0x7f7738013510 tx=0x7f773800b040).stop 2023-11-17T05:34:37.556 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:37.556+0000 7f7749d24700 1 -- 172.21.15.66:0/1227490212 shutdown_connections 2023-11-17T05:34:37.556 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:37.556+0000 7f7749d24700 1 --2- 172.21.15.66:0/1227490212 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f773402ff60 0x7f7734032420 unknown :-1 s=CLOSED pgs=174 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:34:37.556 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:37.556+0000 7f7749d24700 1 --2- 172.21.15.66:0/1227490212 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f77440f51a0 0x7f77441064f0 unknown :-1 s=CLOSED pgs=418 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:34:37.556 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:37.556+0000 7f7749d24700 1 -- 172.21.15.66:0/1227490212 >> 172.21.15.66:0/1227490212 conn(0x7f77440f09a0 msgr2=0x7f77440f13e0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:34:37.556 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:37.556+0000 7f7749d24700 1 -- 172.21.15.66:0/1227490212 shutdown_connections 2023-11-17T05:34:37.556 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:37.556+0000 7f7749d24700 1 -- 172.21.15.66:0/1227490212 wait complete. 2023-11-17T05:34:37.557 INFO:teuthology.orchestra.run.smithi066.stderr:dumped monmap epoch 1 2023-11-17T05:34:37.988 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:34:37 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:34:37.547683+0000 mgr.smithi040.ogcprl (mgr.14164) 414 : cluster [DBG] pgmap v398: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:34:37.989 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:34:37 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:34:37.554769+0000 mon.smithi040 (mon.0) 325 : audit [DBG] from='client.? 172.21.15.66:0/1227490212' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-11-17T05:34:38.977 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2023-11-17T05:34:38.978 DEBUG:teuthology.orchestra.run.smithi066:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.4 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ed6470e2-8508-11ee-8dbe-212e2dc638e7 -- ceph mon dump -f json 2023-11-17T05:34:40.157 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:40.155+0000 7f9613299700 1 -- 172.21.15.66:0/2501084724 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f960c0f5410 msgr2=0x7f960c0f5830 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:34:40.157 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:40.155+0000 7f9613299700 1 --2- 172.21.15.66:0/2501084724 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f960c0f5410 0x7f960c0f5830 secure :-1 s=READY pgs=419 cs=0 l=1 rev1=1 rx=0x7f95fc009fb0 tx=0x7f95fc009b30).stop 2023-11-17T05:34:40.157 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:40.155+0000 7f9613299700 1 -- 172.21.15.66:0/2501084724 shutdown_connections 2023-11-17T05:34:40.157 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:40.155+0000 7f9613299700 1 --2- 172.21.15.66:0/2501084724 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f960c0f5410 0x7f960c0f5830 unknown :-1 s=CLOSED pgs=419 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:34:40.157 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:40.155+0000 7f9613299700 1 -- 172.21.15.66:0/2501084724 >> 172.21.15.66:0/2501084724 conn(0x7f960c0f0b70 msgr2=0x7f960c0f2f90 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:34:40.157 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:40.155+0000 7f9613299700 1 -- 172.21.15.66:0/2501084724 shutdown_connections 2023-11-17T05:34:40.157 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:40.155+0000 7f9613299700 1 -- 172.21.15.66:0/2501084724 wait complete. 2023-11-17T05:34:40.157 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:40.156+0000 7f9613299700 1 Processor -- start 2023-11-17T05:34:40.157 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:40.156+0000 7f9613299700 1 -- start start 2023-11-17T05:34:40.157 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:40.156+0000 7f9613299700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f960c0f5410 0x7f960c0fd3a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:34:40.158 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:40.156+0000 7f9613299700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f960c0fb820 con 0x7f960c0f5410 2023-11-17T05:34:40.158 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:40.156+0000 7f9611035700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f960c0f5410 0x7f960c0fd3a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:34:40.158 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:40.156+0000 7f9611035700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f960c0f5410 0x7f960c0fd3a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.66:33174/0 (socket says 172.21.15.66:33174) 2023-11-17T05:34:40.158 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:40.156+0000 7f9611035700 1 -- 172.21.15.66:0/1031901125 learned_addr learned my addr 172.21.15.66:0/1031901125 (peer_addr_for_me v2:172.21.15.66:0/0) 2023-11-17T05:34:40.158 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:40.156+0000 7f9611035700 1 -- 172.21.15.66:0/1031901125 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f95fc005040 con 0x7f960c0f5410 2023-11-17T05:34:40.158 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:40.157+0000 7f9611035700 1 --2- 172.21.15.66:0/1031901125 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f960c0f5410 0x7f960c0fd3a0 secure :-1 s=READY pgs=420 cs=0 l=1 rev1=1 rx=0x7f95fc013510 tx=0x7f95fc00b040).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:34:40.159 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:40.157+0000 7f9601ffb700 1 -- 172.21.15.66:0/1031901125 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f95fc009320 con 0x7f960c0f5410 2023-11-17T05:34:40.159 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:40.157+0000 7f9613299700 1 -- 172.21.15.66:0/1031901125 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f960c0fb9c0 con 0x7f960c0f5410 2023-11-17T05:34:40.159 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:40.157+0000 7f9601ffb700 1 -- 172.21.15.66:0/1031901125 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(25 keys) v1 ==== 1074+0+0 (secure 0 0 0) 0x7f95fc013f00 con 0x7f960c0f5410 2023-11-17T05:34:40.159 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:40.157+0000 7f9601ffb700 1 -- 172.21.15.66:0/1031901125 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f95fc014d90 con 0x7f960c0f5410 2023-11-17T05:34:40.160 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:40.157+0000 7f9613299700 1 -- 172.21.15.66:0/1031901125 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f960c0fbe50 con 0x7f960c0f5410 2023-11-17T05:34:40.160 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:40.158+0000 7f9601ffb700 1 -- 172.21.15.66:0/1031901125 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 17) v1 ==== 36269+0+0 (secure 0 0 0) 0x7f95fc0095f0 con 0x7f960c0f5410 2023-11-17T05:34:40.160 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:40.158+0000 7f9613299700 1 -- 172.21.15.66:0/1031901125 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f960c059100 con 0x7f960c0f5410 2023-11-17T05:34:40.160 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:40.158+0000 7f9601ffb700 1 --2- 172.21.15.66:0/1031901125 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f95f802ff60 0x7f95f8032420 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:34:40.160 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:40.158+0000 7f9601ffb700 1 -- 172.21.15.66:0/1031901125 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(5..5 src has 1..5) v4 ==== 1270+0+0 (secure 0 0 0) 0x7f95fc041000 con 0x7f960c0f5410 2023-11-17T05:34:40.160 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:40.158+0000 7f9610834700 1 --2- 172.21.15.66:0/1031901125 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f95f802ff60 0x7f95f8032420 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:34:40.160 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:40.159+0000 7f9610834700 1 --2- 172.21.15.66:0/1031901125 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f95f802ff60 0x7f95f8032420 secure :-1 s=READY pgs=175 cs=0 l=1 rev1=1 rx=0x7f96080096a0 tx=0x7f9608006b40).ready entity=mgr.14164 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:34:40.163 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:40.163+0000 7f9601ffb700 1 -- 172.21.15.66:0/1031901125 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+152212 (secure 0 0 0) 0x7f95fc014420 con 0x7f960c0f5410 2023-11-17T05:34:40.386 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:40.385+0000 7f9613299700 1 -- 172.21.15.66:0/1031901125 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "mon dump", "format": "json"} v 0) v1 -- 0x7f960c0fae60 con 0x7f960c0f5410 2023-11-17T05:34:40.386 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:40.386+0000 7f9601ffb700 1 -- 172.21.15.66:0/1031901125 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mon_command_ack([{"prefix": "mon dump", "format": "json"}]=0 dumped monmap epoch 1 v1) v1 ==== 95+0+690 (secure 0 0 0) 0x7f95fc01d7f0 con 0x7f960c0f5410 2023-11-17T05:34:40.386 INFO:teuthology.orchestra.run.smithi066.stdout: 2023-11-17T05:34:40.386 INFO:teuthology.orchestra.run.smithi066.stdout:{"epoch":1,"fsid":"ed6470e2-8508-11ee-8dbe-212e2dc638e7","modified":"2023-11-17T05:20:13.934038Z","created":"2023-11-17T05:20:13.934038Z","min_mon_release":16,"min_mon_release_name":"pacific","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging"],"optional":[]},"mons":[{"rank":0,"name":"smithi040","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:3300","nonce":0},{"type":"v1","addr":"172.21.15.40:6789","nonce":0}]},"addr":"172.21.15.40:6789/0","public_addr":"172.21.15.40:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2023-11-17T05:34:40.388 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:40.388+0000 7f9613299700 1 -- 172.21.15.66:0/1031901125 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f95f802ff60 msgr2=0x7f95f8032420 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:34:40.388 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:40.388+0000 7f9613299700 1 --2- 172.21.15.66:0/1031901125 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f95f802ff60 0x7f95f8032420 secure :-1 s=READY pgs=175 cs=0 l=1 rev1=1 rx=0x7f96080096a0 tx=0x7f9608006b40).stop 2023-11-17T05:34:40.388 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:40.388+0000 7f9613299700 1 -- 172.21.15.66:0/1031901125 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f960c0f5410 msgr2=0x7f960c0fd3a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:34:40.388 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:40.388+0000 7f9613299700 1 --2- 172.21.15.66:0/1031901125 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f960c0f5410 0x7f960c0fd3a0 secure :-1 s=READY pgs=420 cs=0 l=1 rev1=1 rx=0x7f95fc013510 tx=0x7f95fc00b040).stop 2023-11-17T05:34:40.388 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:40.388+0000 7f9613299700 1 -- 172.21.15.66:0/1031901125 shutdown_connections 2023-11-17T05:34:40.388 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:40.388+0000 7f9613299700 1 --2- 172.21.15.66:0/1031901125 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f95f802ff60 0x7f95f8032420 unknown :-1 s=CLOSED pgs=175 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:34:40.389 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:40.388+0000 7f9613299700 1 --2- 172.21.15.66:0/1031901125 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f960c0f5410 0x7f960c0fd3a0 unknown :-1 s=CLOSED pgs=420 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:34:40.389 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:40.388+0000 7f9613299700 1 -- 172.21.15.66:0/1031901125 >> 172.21.15.66:0/1031901125 conn(0x7f960c0f0b70 msgr2=0x7f960c05fe00 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:34:40.389 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:40.388+0000 7f9613299700 1 -- 172.21.15.66:0/1031901125 shutdown_connections 2023-11-17T05:34:40.389 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:40.388+0000 7f9613299700 1 -- 172.21.15.66:0/1031901125 wait complete. 2023-11-17T05:34:40.389 INFO:teuthology.orchestra.run.smithi066.stderr:dumped monmap epoch 1 2023-11-17T05:34:40.989 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:34:40 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:34:39.548024+0000 mgr.smithi040.ogcprl (mgr.14164) 415 : cluster [DBG] pgmap v399: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:34:40.989 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:34:40 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:34:40.386686+0000 mon.smithi040 (mon.0) 326 : audit [DBG] from='client.? 172.21.15.66:0/1031901125' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-11-17T05:34:41.780 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2023-11-17T05:34:41.780 DEBUG:teuthology.orchestra.run.smithi066:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.4 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ed6470e2-8508-11ee-8dbe-212e2dc638e7 -- ceph mon dump -f json 2023-11-17T05:34:41.988 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:34:41 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:34:41.548327+0000 mgr.smithi040.ogcprl (mgr.14164) 416 : cluster [DBG] pgmap v400: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:34:42.955 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:42.951+0000 7fd0e16c0700 1 -- 172.21.15.66:0/1008225532 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fd0dc0f62a0 msgr2=0x7fd0dc0f66c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:34:42.955 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:42.951+0000 7fd0e16c0700 1 --2- 172.21.15.66:0/1008225532 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fd0dc0f62a0 0x7fd0dc0f66c0 secure :-1 s=READY pgs=421 cs=0 l=1 rev1=1 rx=0x7fd0cc009fb0 tx=0x7fd0cc009b30).stop 2023-11-17T05:34:42.956 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:42.951+0000 7fd0e16c0700 1 -- 172.21.15.66:0/1008225532 shutdown_connections 2023-11-17T05:34:42.956 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:42.951+0000 7fd0e16c0700 1 --2- 172.21.15.66:0/1008225532 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fd0dc0f62a0 0x7fd0dc0f66c0 unknown :-1 s=CLOSED pgs=421 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:34:42.956 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:42.951+0000 7fd0e16c0700 1 -- 172.21.15.66:0/1008225532 >> 172.21.15.66:0/1008225532 conn(0x7fd0dc0f1aa0 msgr2=0x7fd0dc0f3ec0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:34:42.956 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:42.951+0000 7fd0e16c0700 1 -- 172.21.15.66:0/1008225532 shutdown_connections 2023-11-17T05:34:42.956 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:42.951+0000 7fd0e16c0700 1 -- 172.21.15.66:0/1008225532 wait complete. 2023-11-17T05:34:42.956 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:42.951+0000 7fd0e16c0700 1 Processor -- start 2023-11-17T05:34:42.957 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:42.952+0000 7fd0e16c0700 1 -- start start 2023-11-17T05:34:42.957 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:42.952+0000 7fd0e16c0700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fd0dc0f62a0 0x7fd0dc1064b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:34:42.957 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:42.952+0000 7fd0e16c0700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fd0dc1069a0 con 0x7fd0dc0f62a0 2023-11-17T05:34:42.957 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:42.952+0000 7fd0dbfff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fd0dc0f62a0 0x7fd0dc1064b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:34:42.957 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:42.952+0000 7fd0dbfff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fd0dc0f62a0 0x7fd0dc1064b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.66:52466/0 (socket says 172.21.15.66:52466) 2023-11-17T05:34:42.957 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:42.952+0000 7fd0dbfff700 1 -- 172.21.15.66:0/850544330 learned_addr learned my addr 172.21.15.66:0/850544330 (peer_addr_for_me v2:172.21.15.66:0/0) 2023-11-17T05:34:42.958 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:42.952+0000 7fd0dbfff700 1 -- 172.21.15.66:0/850544330 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fd0cc005040 con 0x7fd0dc0f62a0 2023-11-17T05:34:42.958 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:42.952+0000 7fd0dbfff700 1 --2- 172.21.15.66:0/850544330 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fd0dc0f62a0 0x7fd0dc1064b0 secure :-1 s=READY pgs=422 cs=0 l=1 rev1=1 rx=0x7fd0cc013510 tx=0x7fd0cc00b040).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:34:42.958 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:42.953+0000 7fd0d8ff9700 1 -- 172.21.15.66:0/850544330 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7fd0cc009320 con 0x7fd0dc0f62a0 2023-11-17T05:34:42.958 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:42.953+0000 7fd0e16c0700 1 -- 172.21.15.66:0/850544330 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fd0dc106ba0 con 0x7fd0dc0f62a0 2023-11-17T05:34:42.958 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:42.953+0000 7fd0d8ff9700 1 -- 172.21.15.66:0/850544330 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(25 keys) v1 ==== 1074+0+0 (secure 0 0 0) 0x7fd0cc013f00 con 0x7fd0dc0f62a0 2023-11-17T05:34:42.959 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:42.953+0000 7fd0d8ff9700 1 -- 172.21.15.66:0/850544330 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7fd0cc014dc0 con 0x7fd0dc0f62a0 2023-11-17T05:34:42.976 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:42.953+0000 7fd0e16c0700 1 -- 172.21.15.66:0/850544330 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fd0dc107030 con 0x7fd0dc0f62a0 2023-11-17T05:34:42.976 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:42.954+0000 7fd0d8ff9700 1 -- 172.21.15.66:0/850544330 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 17) v1 ==== 36269+0+0 (secure 0 0 0) 0x7fd0cc014420 con 0x7fd0dc0f62a0 2023-11-17T05:34:42.976 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:42.954+0000 7fd0e16c0700 1 -- 172.21.15.66:0/850544330 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fd0dc045ad0 con 0x7fd0dc0f62a0 2023-11-17T05:34:42.976 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:42.954+0000 7fd0d8ff9700 1 --2- 172.21.15.66:0/850544330 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fd0c402ff60 0x7fd0c4032420 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:34:42.976 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:42.954+0000 7fd0d8ff9700 1 -- 172.21.15.66:0/850544330 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(5..5 src has 1..5) v4 ==== 1270+0+0 (secure 0 0 0) 0x7fd0cc0410d0 con 0x7fd0dc0f62a0 2023-11-17T05:34:42.977 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:42.954+0000 7fd0db7fe700 1 --2- 172.21.15.66:0/850544330 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fd0c402ff60 0x7fd0c4032420 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:34:42.977 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:42.954+0000 7fd0db7fe700 1 --2- 172.21.15.66:0/850544330 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fd0c402ff60 0x7fd0c4032420 secure :-1 s=READY pgs=176 cs=0 l=1 rev1=1 rx=0x7fd0d00096a0 tx=0x7fd0d0006b40).ready entity=mgr.14164 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:34:42.978 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:42.959+0000 7fd0d8ff9700 1 -- 172.21.15.66:0/850544330 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+152212 (secure 0 0 0) 0x7fd0cc0095f0 con 0x7fd0dc0f62a0 2023-11-17T05:34:43.179 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:43.179+0000 7fd0e16c0700 1 -- 172.21.15.66:0/850544330 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "mon dump", "format": "json"} v 0) v1 -- 0x7fd0dc107ee0 con 0x7fd0dc0f62a0 2023-11-17T05:34:43.180 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:43.179+0000 7fd0d8ff9700 1 -- 172.21.15.66:0/850544330 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mon_command_ack([{"prefix": "mon dump", "format": "json"}]=0 dumped monmap epoch 1 v1) v1 ==== 95+0+690 (secure 0 0 0) 0x7fd0cc01cb10 con 0x7fd0dc0f62a0 2023-11-17T05:34:43.180 INFO:teuthology.orchestra.run.smithi066.stdout: 2023-11-17T05:34:43.180 INFO:teuthology.orchestra.run.smithi066.stdout:{"epoch":1,"fsid":"ed6470e2-8508-11ee-8dbe-212e2dc638e7","modified":"2023-11-17T05:20:13.934038Z","created":"2023-11-17T05:20:13.934038Z","min_mon_release":16,"min_mon_release_name":"pacific","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging"],"optional":[]},"mons":[{"rank":0,"name":"smithi040","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:3300","nonce":0},{"type":"v1","addr":"172.21.15.40:6789","nonce":0}]},"addr":"172.21.15.40:6789/0","public_addr":"172.21.15.40:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2023-11-17T05:34:43.181 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:43.181+0000 7fd0e16c0700 1 -- 172.21.15.66:0/850544330 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fd0c402ff60 msgr2=0x7fd0c4032420 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:34:43.181 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:43.181+0000 7fd0e16c0700 1 --2- 172.21.15.66:0/850544330 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fd0c402ff60 0x7fd0c4032420 secure :-1 s=READY pgs=176 cs=0 l=1 rev1=1 rx=0x7fd0d00096a0 tx=0x7fd0d0006b40).stop 2023-11-17T05:34:43.181 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:43.181+0000 7fd0e16c0700 1 -- 172.21.15.66:0/850544330 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fd0dc0f62a0 msgr2=0x7fd0dc1064b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:34:43.181 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:43.181+0000 7fd0e16c0700 1 --2- 172.21.15.66:0/850544330 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fd0dc0f62a0 0x7fd0dc1064b0 secure :-1 s=READY pgs=422 cs=0 l=1 rev1=1 rx=0x7fd0cc013510 tx=0x7fd0cc00b040).stop 2023-11-17T05:34:43.182 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:43.181+0000 7fd0e16c0700 1 -- 172.21.15.66:0/850544330 shutdown_connections 2023-11-17T05:34:43.182 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:43.181+0000 7fd0e16c0700 1 --2- 172.21.15.66:0/850544330 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fd0c402ff60 0x7fd0c4032420 unknown :-1 s=CLOSED pgs=176 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:34:43.182 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:43.181+0000 7fd0e16c0700 1 --2- 172.21.15.66:0/850544330 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fd0dc0f62a0 0x7fd0dc1064b0 unknown :-1 s=CLOSED pgs=422 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:34:43.182 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:43.181+0000 7fd0e16c0700 1 -- 172.21.15.66:0/850544330 >> 172.21.15.66:0/850544330 conn(0x7fd0dc0f1aa0 msgr2=0x7fd0dc0f2510 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:34:43.182 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:43.181+0000 7fd0e16c0700 1 -- 172.21.15.66:0/850544330 shutdown_connections 2023-11-17T05:34:43.182 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:43.181+0000 7fd0e16c0700 1 -- 172.21.15.66:0/850544330 wait complete. 2023-11-17T05:34:43.182 INFO:teuthology.orchestra.run.smithi066.stderr:dumped monmap epoch 1 2023-11-17T05:34:43.488 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:34:43 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:34:43.180080+0000 mon.smithi040 (mon.0) 327 : audit [DBG] from='client.? 172.21.15.66:0/850544330' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-11-17T05:34:44.488 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:34:44 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:34:43.548730+0000 mgr.smithi040.ogcprl (mgr.14164) 417 : cluster [DBG] pgmap v401: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:34:44.557 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2023-11-17T05:34:44.557 DEBUG:teuthology.orchestra.run.smithi066:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.4 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ed6470e2-8508-11ee-8dbe-212e2dc638e7 -- ceph mon dump -f json 2023-11-17T05:34:45.800 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:45.800+0000 7fba1db04700 1 -- 172.21.15.66:0/1094785515 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fba18060280 msgr2=0x7fba180f90f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:34:45.801 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:45.800+0000 7fba1db04700 1 --2- 172.21.15.66:0/1094785515 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fba18060280 0x7fba180f90f0 secure :-1 s=READY pgs=423 cs=0 l=1 rev1=1 rx=0x7fba08009fb0 tx=0x7fba08009b30).stop 2023-11-17T05:34:45.801 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:45.800+0000 7fba1db04700 1 -- 172.21.15.66:0/1094785515 shutdown_connections 2023-11-17T05:34:45.801 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:45.800+0000 7fba1db04700 1 --2- 172.21.15.66:0/1094785515 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fba18060280 0x7fba180f90f0 unknown :-1 s=CLOSED pgs=423 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:34:45.801 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:45.800+0000 7fba1db04700 1 -- 172.21.15.66:0/1094785515 >> 172.21.15.66:0/1094785515 conn(0x7fba180f0b70 msgr2=0x7fba180f2f90 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:34:45.801 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:45.800+0000 7fba1db04700 1 -- 172.21.15.66:0/1094785515 shutdown_connections 2023-11-17T05:34:45.801 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:45.800+0000 7fba1db04700 1 -- 172.21.15.66:0/1094785515 wait complete. 2023-11-17T05:34:45.801 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:45.800+0000 7fba1db04700 1 Processor -- start 2023-11-17T05:34:45.803 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:45.801+0000 7fba1db04700 1 -- start start 2023-11-17T05:34:45.803 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:45.801+0000 7fba1db04700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fba18060280 0x7fba180f6b70 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:34:45.803 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:45.801+0000 7fba1db04700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fba180f4ff0 con 0x7fba18060280 2023-11-17T05:34:45.803 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:45.801+0000 7fba177fe700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fba18060280 0x7fba180f6b70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:34:45.803 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:45.801+0000 7fba177fe700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fba18060280 0x7fba180f6b70 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.66:52490/0 (socket says 172.21.15.66:52490) 2023-11-17T05:34:45.803 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:45.801+0000 7fba177fe700 1 -- 172.21.15.66:0/2089847755 learned_addr learned my addr 172.21.15.66:0/2089847755 (peer_addr_for_me v2:172.21.15.66:0/0) 2023-11-17T05:34:45.804 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:45.801+0000 7fba177fe700 1 -- 172.21.15.66:0/2089847755 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fba08005040 con 0x7fba18060280 2023-11-17T05:34:45.804 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:45.802+0000 7fba177fe700 1 --2- 172.21.15.66:0/2089847755 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fba18060280 0x7fba180f6b70 secure :-1 s=READY pgs=424 cs=0 l=1 rev1=1 rx=0x7fba08013510 tx=0x7fba0800b040).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:34:45.804 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:45.802+0000 7fba0ffff700 1 -- 172.21.15.66:0/2089847755 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7fba08009320 con 0x7fba18060280 2023-11-17T05:34:45.804 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:45.802+0000 7fba1db04700 1 -- 172.21.15.66:0/2089847755 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fba180f5190 con 0x7fba18060280 2023-11-17T05:34:45.804 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:45.802+0000 7fba0ffff700 1 -- 172.21.15.66:0/2089847755 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(25 keys) v1 ==== 1074+0+0 (secure 0 0 0) 0x7fba08013f00 con 0x7fba18060280 2023-11-17T05:34:45.804 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:45.802+0000 7fba0ffff700 1 -- 172.21.15.66:0/2089847755 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7fba08014dc0 con 0x7fba18060280 2023-11-17T05:34:45.804 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:45.802+0000 7fba1db04700 1 -- 172.21.15.66:0/2089847755 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fba180f5620 con 0x7fba18060280 2023-11-17T05:34:45.804 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:45.803+0000 7fba0ffff700 1 -- 172.21.15.66:0/2089847755 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 17) v1 ==== 36269+0+0 (secure 0 0 0) 0x7fba080095f0 con 0x7fba18060280 2023-11-17T05:34:45.804 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:45.803+0000 7fba1db04700 1 -- 172.21.15.66:0/2089847755 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fba18059100 con 0x7fba18060280 2023-11-17T05:34:45.804 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:45.803+0000 7fba0ffff700 1 --2- 172.21.15.66:0/2089847755 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fba0402ff60 0x7fba04032420 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:34:45.804 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:45.803+0000 7fba0ffff700 1 -- 172.21.15.66:0/2089847755 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(5..5 src has 1..5) v4 ==== 1270+0+0 (secure 0 0 0) 0x7fba08041000 con 0x7fba18060280 2023-11-17T05:34:45.804 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:45.803+0000 7fba16ffd700 1 --2- 172.21.15.66:0/2089847755 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fba0402ff60 0x7fba04032420 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:34:45.805 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:45.803+0000 7fba16ffd700 1 --2- 172.21.15.66:0/2089847755 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fba0402ff60 0x7fba04032420 secure :-1 s=READY pgs=177 cs=0 l=1 rev1=1 rx=0x7fba000096a0 tx=0x7fba00006b40).ready entity=mgr.14164 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:34:45.808 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:45.808+0000 7fba0ffff700 1 -- 172.21.15.66:0/2089847755 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+152212 (secure 0 0 0) 0x7fba08014420 con 0x7fba18060280 2023-11-17T05:34:45.989 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:34:45 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:34:45.549067+0000 mgr.smithi040.ogcprl (mgr.14164) 418 : cluster [DBG] pgmap v402: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:34:46.027 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:46.027+0000 7fba1db04700 1 -- 172.21.15.66:0/2089847755 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "mon dump", "format": "json"} v 0) v1 -- 0x7fba180fbf50 con 0x7fba18060280 2023-11-17T05:34:46.028 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:46.028+0000 7fba0ffff700 1 -- 172.21.15.66:0/2089847755 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mon_command_ack([{"prefix": "mon dump", "format": "json"}]=0 dumped monmap epoch 1 v1) v1 ==== 95+0+690 (secure 0 0 0) 0x7fba08010230 con 0x7fba18060280 2023-11-17T05:34:46.028 INFO:teuthology.orchestra.run.smithi066.stdout: 2023-11-17T05:34:46.028 INFO:teuthology.orchestra.run.smithi066.stdout:{"epoch":1,"fsid":"ed6470e2-8508-11ee-8dbe-212e2dc638e7","modified":"2023-11-17T05:20:13.934038Z","created":"2023-11-17T05:20:13.934038Z","min_mon_release":16,"min_mon_release_name":"pacific","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging"],"optional":[]},"mons":[{"rank":0,"name":"smithi040","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:3300","nonce":0},{"type":"v1","addr":"172.21.15.40:6789","nonce":0}]},"addr":"172.21.15.40:6789/0","public_addr":"172.21.15.40:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2023-11-17T05:34:46.030 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:46.029+0000 7fba1db04700 1 -- 172.21.15.66:0/2089847755 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fba0402ff60 msgr2=0x7fba04032420 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:34:46.030 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:46.029+0000 7fba1db04700 1 --2- 172.21.15.66:0/2089847755 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fba0402ff60 0x7fba04032420 secure :-1 s=READY pgs=177 cs=0 l=1 rev1=1 rx=0x7fba000096a0 tx=0x7fba00006b40).stop 2023-11-17T05:34:46.030 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:46.029+0000 7fba1db04700 1 -- 172.21.15.66:0/2089847755 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fba18060280 msgr2=0x7fba180f6b70 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:34:46.030 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:46.029+0000 7fba1db04700 1 --2- 172.21.15.66:0/2089847755 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fba18060280 0x7fba180f6b70 secure :-1 s=READY pgs=424 cs=0 l=1 rev1=1 rx=0x7fba08013510 tx=0x7fba0800b040).stop 2023-11-17T05:34:46.030 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:46.030+0000 7fba1db04700 1 -- 172.21.15.66:0/2089847755 shutdown_connections 2023-11-17T05:34:46.030 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:46.030+0000 7fba1db04700 1 --2- 172.21.15.66:0/2089847755 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fba0402ff60 0x7fba04032420 unknown :-1 s=CLOSED pgs=177 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:34:46.030 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:46.030+0000 7fba1db04700 1 --2- 172.21.15.66:0/2089847755 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fba18060280 0x7fba180f6b70 unknown :-1 s=CLOSED pgs=424 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:34:46.030 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:46.030+0000 7fba1db04700 1 -- 172.21.15.66:0/2089847755 >> 172.21.15.66:0/2089847755 conn(0x7fba180f0b70 msgr2=0x7fba180f1660 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:34:46.031 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:46.030+0000 7fba1db04700 1 -- 172.21.15.66:0/2089847755 shutdown_connections 2023-11-17T05:34:46.031 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:46.030+0000 7fba1db04700 1 -- 172.21.15.66:0/2089847755 wait complete. 2023-11-17T05:34:46.033 INFO:teuthology.orchestra.run.smithi066.stderr:dumped monmap epoch 1 2023-11-17T05:34:46.988 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:34:46 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:34:46.028438+0000 mon.smithi040 (mon.0) 328 : audit [DBG] from='client.? 172.21.15.66:0/2089847755' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-11-17T05:34:47.423 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2023-11-17T05:34:47.423 DEBUG:teuthology.orchestra.run.smithi066:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.4 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ed6470e2-8508-11ee-8dbe-212e2dc638e7 -- ceph mon dump -f json 2023-11-17T05:34:47.988 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:34:47 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:34:47.549443+0000 mgr.smithi040.ogcprl (mgr.14164) 419 : cluster [DBG] pgmap v403: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:34:48.621 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:48.621+0000 7fdeed5a9700 1 -- 172.21.15.66:0/2273979387 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fdee8060280 msgr2=0x7fdee80f90f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:34:48.622 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:48.621+0000 7fdeed5a9700 1 --2- 172.21.15.66:0/2273979387 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fdee8060280 0x7fdee80f90f0 secure :-1 s=READY pgs=425 cs=0 l=1 rev1=1 rx=0x7fdedc009fb0 tx=0x7fdedc009b30).stop 2023-11-17T05:34:48.622 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:48.621+0000 7fdeed5a9700 1 -- 172.21.15.66:0/2273979387 shutdown_connections 2023-11-17T05:34:48.622 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:48.621+0000 7fdeed5a9700 1 --2- 172.21.15.66:0/2273979387 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fdee8060280 0x7fdee80f90f0 unknown :-1 s=CLOSED pgs=425 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:34:48.622 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:48.621+0000 7fdeed5a9700 1 -- 172.21.15.66:0/2273979387 >> 172.21.15.66:0/2273979387 conn(0x7fdee80f0b70 msgr2=0x7fdee80f2f90 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:34:48.622 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:48.621+0000 7fdeed5a9700 1 -- 172.21.15.66:0/2273979387 shutdown_connections 2023-11-17T05:34:48.622 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:48.621+0000 7fdeed5a9700 1 -- 172.21.15.66:0/2273979387 wait complete. 2023-11-17T05:34:48.623 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:48.622+0000 7fdeed5a9700 1 Processor -- start 2023-11-17T05:34:48.624 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:48.622+0000 7fdeed5a9700 1 -- start start 2023-11-17T05:34:48.624 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:48.622+0000 7fdeed5a9700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fdee8060280 0x7fdee80f6b70 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:34:48.624 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:48.622+0000 7fdeed5a9700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fdee80f4ff0 con 0x7fdee8060280 2023-11-17T05:34:48.624 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:48.622+0000 7fdee6ffd700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fdee8060280 0x7fdee80f6b70 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:34:48.624 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:48.622+0000 7fdee6ffd700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fdee8060280 0x7fdee80f6b70 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.66:52518/0 (socket says 172.21.15.66:52518) 2023-11-17T05:34:48.624 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:48.622+0000 7fdee6ffd700 1 -- 172.21.15.66:0/1719113583 learned_addr learned my addr 172.21.15.66:0/1719113583 (peer_addr_for_me v2:172.21.15.66:0/0) 2023-11-17T05:34:48.624 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:48.623+0000 7fdee6ffd700 1 -- 172.21.15.66:0/1719113583 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fdedc005040 con 0x7fdee8060280 2023-11-17T05:34:48.624 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:48.623+0000 7fdee6ffd700 1 --2- 172.21.15.66:0/1719113583 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fdee8060280 0x7fdee80f6b70 secure :-1 s=READY pgs=426 cs=0 l=1 rev1=1 rx=0x7fdedc013510 tx=0x7fdedc00b040).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:34:48.624 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:48.623+0000 7fdecf7fe700 1 -- 172.21.15.66:0/1719113583 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7fdedc009320 con 0x7fdee8060280 2023-11-17T05:34:48.624 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:48.623+0000 7fdeed5a9700 1 -- 172.21.15.66:0/1719113583 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fdee80f5190 con 0x7fdee8060280 2023-11-17T05:34:48.624 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:48.623+0000 7fdecf7fe700 1 -- 172.21.15.66:0/1719113583 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(25 keys) v1 ==== 1074+0+0 (secure 0 0 0) 0x7fdedc013f00 con 0x7fdee8060280 2023-11-17T05:34:48.624 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:48.623+0000 7fdecf7fe700 1 -- 172.21.15.66:0/1719113583 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7fdedc014dc0 con 0x7fdee8060280 2023-11-17T05:34:48.624 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:48.623+0000 7fdeed5a9700 1 -- 172.21.15.66:0/1719113583 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fdee80f5620 con 0x7fdee8060280 2023-11-17T05:34:48.628 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:48.624+0000 7fdeed5a9700 1 -- 172.21.15.66:0/1719113583 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fdee8059100 con 0x7fdee8060280 2023-11-17T05:34:48.628 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:48.624+0000 7fdecf7fe700 1 -- 172.21.15.66:0/1719113583 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 17) v1 ==== 36269+0+0 (secure 0 0 0) 0x7fdedc014420 con 0x7fdee8060280 2023-11-17T05:34:48.628 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:48.624+0000 7fdecf7fe700 1 --2- 172.21.15.66:0/1719113583 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fded002ff60 0x7fded0032420 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:34:48.628 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:48.624+0000 7fdecf7fe700 1 -- 172.21.15.66:0/1719113583 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(5..5 src has 1..5) v4 ==== 1270+0+0 (secure 0 0 0) 0x7fdedc041050 con 0x7fdee8060280 2023-11-17T05:34:48.628 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:48.624+0000 7fdee67fc700 1 --2- 172.21.15.66:0/1719113583 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fded002ff60 0x7fded0032420 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:34:48.628 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:48.625+0000 7fdee67fc700 1 --2- 172.21.15.66:0/1719113583 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fded002ff60 0x7fded0032420 secure :-1 s=READY pgs=178 cs=0 l=1 rev1=1 rx=0x7fded80096a0 tx=0x7fded8006b40).ready entity=mgr.14164 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:34:48.628 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:48.628+0000 7fdecf7fe700 1 -- 172.21.15.66:0/1719113583 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+152212 (secure 0 0 0) 0x7fdedc0095f0 con 0x7fdee8060280 2023-11-17T05:34:48.850 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:48.850+0000 7fdeed5a9700 1 -- 172.21.15.66:0/1719113583 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "mon dump", "format": "json"} v 0) v1 -- 0x7fdee80fbf50 con 0x7fdee8060280 2023-11-17T05:34:48.851 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:48.850+0000 7fdecf7fe700 1 -- 172.21.15.66:0/1719113583 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mon_command_ack([{"prefix": "mon dump", "format": "json"}]=0 dumped monmap epoch 1 v1) v1 ==== 95+0+690 (secure 0 0 0) 0x7fdedc01cb10 con 0x7fdee8060280 2023-11-17T05:34:48.851 INFO:teuthology.orchestra.run.smithi066.stdout: 2023-11-17T05:34:48.851 INFO:teuthology.orchestra.run.smithi066.stdout:{"epoch":1,"fsid":"ed6470e2-8508-11ee-8dbe-212e2dc638e7","modified":"2023-11-17T05:20:13.934038Z","created":"2023-11-17T05:20:13.934038Z","min_mon_release":16,"min_mon_release_name":"pacific","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging"],"optional":[]},"mons":[{"rank":0,"name":"smithi040","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:3300","nonce":0},{"type":"v1","addr":"172.21.15.40:6789","nonce":0}]},"addr":"172.21.15.40:6789/0","public_addr":"172.21.15.40:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2023-11-17T05:34:48.852 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:48.852+0000 7fdeed5a9700 1 -- 172.21.15.66:0/1719113583 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fded002ff60 msgr2=0x7fded0032420 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:34:48.853 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:48.852+0000 7fdeed5a9700 1 --2- 172.21.15.66:0/1719113583 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fded002ff60 0x7fded0032420 secure :-1 s=READY pgs=178 cs=0 l=1 rev1=1 rx=0x7fded80096a0 tx=0x7fded8006b40).stop 2023-11-17T05:34:48.853 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:48.852+0000 7fdeed5a9700 1 -- 172.21.15.66:0/1719113583 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fdee8060280 msgr2=0x7fdee80f6b70 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:34:48.853 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:48.852+0000 7fdeed5a9700 1 --2- 172.21.15.66:0/1719113583 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fdee8060280 0x7fdee80f6b70 secure :-1 s=READY pgs=426 cs=0 l=1 rev1=1 rx=0x7fdedc013510 tx=0x7fdedc00b040).stop 2023-11-17T05:34:48.853 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:48.852+0000 7fdeed5a9700 1 -- 172.21.15.66:0/1719113583 shutdown_connections 2023-11-17T05:34:48.853 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:48.852+0000 7fdeed5a9700 1 --2- 172.21.15.66:0/1719113583 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fded002ff60 0x7fded0032420 unknown :-1 s=CLOSED pgs=178 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:34:48.853 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:48.852+0000 7fdeed5a9700 1 --2- 172.21.15.66:0/1719113583 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fdee8060280 0x7fdee80f6b70 unknown :-1 s=CLOSED pgs=426 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:34:48.853 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:48.852+0000 7fdeed5a9700 1 -- 172.21.15.66:0/1719113583 >> 172.21.15.66:0/1719113583 conn(0x7fdee80f0b70 msgr2=0x7fdee80f1660 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:34:48.853 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:48.852+0000 7fdeed5a9700 1 -- 172.21.15.66:0/1719113583 shutdown_connections 2023-11-17T05:34:48.853 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:48.852+0000 7fdeed5a9700 1 -- 172.21.15.66:0/1719113583 wait complete. 2023-11-17T05:34:48.853 INFO:teuthology.orchestra.run.smithi066.stderr:dumped monmap epoch 1 2023-11-17T05:34:49.239 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:34:48 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:34:48.851206+0000 mon.smithi040 (mon.0) 329 : audit [DBG] from='client.? 172.21.15.66:0/1719113583' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-11-17T05:34:50.232 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2023-11-17T05:34:50.232 DEBUG:teuthology.orchestra.run.smithi066:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.4 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ed6470e2-8508-11ee-8dbe-212e2dc638e7 -- ceph mon dump -f json 2023-11-17T05:34:50.989 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:34:50 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:34:49.549855+0000 mgr.smithi040.ogcprl (mgr.14164) 420 : cluster [DBG] pgmap v404: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:34:51.457 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:51.457+0000 7fd681daa700 1 -- 172.21.15.66:0/1584763664 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fd67c060220 msgr2=0x7fd67c0f9150 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:34:51.457 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:51.457+0000 7fd681daa700 1 --2- 172.21.15.66:0/1584763664 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fd67c060220 0x7fd67c0f9150 secure :-1 s=READY pgs=427 cs=0 l=1 rev1=1 rx=0x7fd66c009fb0 tx=0x7fd66c009b30).stop 2023-11-17T05:34:51.458 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:51.457+0000 7fd681daa700 1 -- 172.21.15.66:0/1584763664 shutdown_connections 2023-11-17T05:34:51.458 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:51.457+0000 7fd681daa700 1 --2- 172.21.15.66:0/1584763664 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fd67c060220 0x7fd67c0f9150 unknown :-1 s=CLOSED pgs=427 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:34:51.458 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:51.457+0000 7fd681daa700 1 -- 172.21.15.66:0/1584763664 >> 172.21.15.66:0/1584763664 conn(0x7fd67c0f0b70 msgr2=0x7fd67c0f2f90 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:34:51.458 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:51.457+0000 7fd681daa700 1 -- 172.21.15.66:0/1584763664 shutdown_connections 2023-11-17T05:34:51.458 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:51.457+0000 7fd681daa700 1 -- 172.21.15.66:0/1584763664 wait complete. 2023-11-17T05:34:51.458 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:51.457+0000 7fd681daa700 1 Processor -- start 2023-11-17T05:34:51.458 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:51.457+0000 7fd681daa700 1 -- start start 2023-11-17T05:34:51.460 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:51.458+0000 7fd681daa700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fd67c060220 0x7fd67c0f6bd0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:34:51.460 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:51.458+0000 7fd681daa700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fd67c0f5050 con 0x7fd67c060220 2023-11-17T05:34:51.460 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:51.458+0000 7fd67b7fe700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fd67c060220 0x7fd67c0f6bd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:34:51.460 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:51.458+0000 7fd67b7fe700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fd67c060220 0x7fd67c0f6bd0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.66:52542/0 (socket says 172.21.15.66:52542) 2023-11-17T05:34:51.460 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:51.458+0000 7fd67b7fe700 1 -- 172.21.15.66:0/2923212451 learned_addr learned my addr 172.21.15.66:0/2923212451 (peer_addr_for_me v2:172.21.15.66:0/0) 2023-11-17T05:34:51.460 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:51.458+0000 7fd67b7fe700 1 -- 172.21.15.66:0/2923212451 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fd66c005040 con 0x7fd67c060220 2023-11-17T05:34:51.460 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:51.458+0000 7fd67b7fe700 1 --2- 172.21.15.66:0/2923212451 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fd67c060220 0x7fd67c0f6bd0 secure :-1 s=READY pgs=428 cs=0 l=1 rev1=1 rx=0x7fd66c013510 tx=0x7fd66c00b040).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:34:51.460 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:51.458+0000 7fd663fff700 1 -- 172.21.15.66:0/2923212451 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7fd66c009320 con 0x7fd67c060220 2023-11-17T05:34:51.460 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:51.458+0000 7fd681daa700 1 -- 172.21.15.66:0/2923212451 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fd67c0f51f0 con 0x7fd67c060220 2023-11-17T05:34:51.460 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:51.458+0000 7fd663fff700 1 -- 172.21.15.66:0/2923212451 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(25 keys) v1 ==== 1074+0+0 (secure 0 0 0) 0x7fd66c013f00 con 0x7fd67c060220 2023-11-17T05:34:51.460 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:51.458+0000 7fd663fff700 1 -- 172.21.15.66:0/2923212451 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7fd66c014dc0 con 0x7fd67c060220 2023-11-17T05:34:51.460 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:51.458+0000 7fd681daa700 1 -- 172.21.15.66:0/2923212451 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fd67c0f5680 con 0x7fd67c060220 2023-11-17T05:34:51.461 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:51.459+0000 7fd663fff700 1 -- 172.21.15.66:0/2923212451 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 17) v1 ==== 36269+0+0 (secure 0 0 0) 0x7fd66c0095f0 con 0x7fd67c060220 2023-11-17T05:34:51.461 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:51.459+0000 7fd681daa700 1 -- 172.21.15.66:0/2923212451 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fd67c059100 con 0x7fd67c060220 2023-11-17T05:34:51.461 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:51.459+0000 7fd663fff700 1 --2- 172.21.15.66:0/2923212451 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fd66402ff60 0x7fd664032420 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:34:51.461 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:51.459+0000 7fd663fff700 1 -- 172.21.15.66:0/2923212451 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(5..5 src has 1..5) v4 ==== 1270+0+0 (secure 0 0 0) 0x7fd66c041040 con 0x7fd67c060220 2023-11-17T05:34:51.461 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:51.460+0000 7fd67affd700 1 --2- 172.21.15.66:0/2923212451 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fd66402ff60 0x7fd664032420 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:34:51.461 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:51.460+0000 7fd67affd700 1 --2- 172.21.15.66:0/2923212451 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fd66402ff60 0x7fd664032420 secure :-1 s=READY pgs=179 cs=0 l=1 rev1=1 rx=0x7fd6700096a0 tx=0x7fd670006b40).ready entity=mgr.14164 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:34:51.464 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:51.464+0000 7fd663fff700 1 -- 172.21.15.66:0/2923212451 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+152212 (secure 0 0 0) 0x7fd66c014420 con 0x7fd67c060220 2023-11-17T05:34:51.683 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:51.683+0000 7fd681daa700 1 -- 172.21.15.66:0/2923212451 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "mon dump", "format": "json"} v 0) v1 -- 0x7fd67c0fbf50 con 0x7fd67c060220 2023-11-17T05:35:05.945 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:51.684+0000 7fd663fff700 1 -- 172.21.15.66:0/2923212451 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mon_command_ack([{"prefix": "mon dump", "format": "json"}]=0 dumped monmap epoch 1 v1) v1 ==== 95+0+690 (secure 0 0 0) 0x7fd66c010230 con 0x7fd67c060220 2023-11-17T05:35:06.045 INFO:teuthology.orchestra.run.smithi066.stdout: 2023-11-17T05:35:06.045 INFO:teuthology.orchestra.run.smithi066.stdout:{"epoch":1,"fsid":"ed6470e2-8508-11ee-8dbe-212e2dc638e7","modified":"2023-11-17T05:20:13.934038Z","created":"2023-11-17T05:20:13.934038Z","min_mon_release":16,"min_mon_release_name":"pacific","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging"],"optional":[]},"mons":[{"rank":0,"name":"smithi040","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:3300","nonce":0},{"type":"v1","addr":"172.21.15.40:6789","nonce":0}]},"addr":"172.21.15.40:6789/0","public_addr":"172.21.15.40:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2023-11-17T05:35:06.046 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:34:51 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:34:51.550210+0000 mgr.smithi040.ogcprl (mgr.14164) 421 : cluster [DBG] pgmap v405: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:35:06.077 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:34:52 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:34:51.684425+0000 mon.smithi040 (mon.0) 330 : audit [DBG] from='client.? 172.21.15.66:0/2923212451' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-11-17T05:35:06.077 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:34:53 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:34:53.550583+0000 mgr.smithi040.ogcprl (mgr.14164) 422 : cluster [DBG] pgmap v406: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:35:06.077 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:34:55 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:34:55.550854+0000 mgr.smithi040.ogcprl (mgr.14164) 423 : cluster [DBG] pgmap v407: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:35:06.077 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:34:57 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:34:57.551121+0000 mgr.smithi040.ogcprl (mgr.14164) 424 : cluster [DBG] pgmap v408: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:35:06.077 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:35:00 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:34:59.551477+0000 mgr.smithi040.ogcprl (mgr.14164) 425 : cluster [DBG] pgmap v409: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:35:06.077 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:35:01 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:35:01.551861+0000 mgr.smithi040.ogcprl (mgr.14164) 426 : cluster [DBG] pgmap v410: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:35:06.077 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:35:03 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:35:03.552159+0000 mgr.smithi040.ogcprl (mgr.14164) 427 : cluster [DBG] pgmap v411: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:35:06.077 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:35:05 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:35:05.552456+0000 mgr.smithi040.ogcprl (mgr.14164) 428 : cluster [DBG] pgmap v412: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:35:06.078 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:51.685+0000 7fd681daa700 1 -- 172.21.15.66:0/2923212451 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fd66402ff60 msgr2=0x7fd664032420 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:35:06.078 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:51.685+0000 7fd681daa700 1 --2- 172.21.15.66:0/2923212451 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fd66402ff60 0x7fd664032420 secure :-1 s=READY pgs=179 cs=0 l=1 rev1=1 rx=0x7fd6700096a0 tx=0x7fd670006b40).stop 2023-11-17T05:35:06.078 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:51.685+0000 7fd681daa700 1 -- 172.21.15.66:0/2923212451 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fd67c060220 msgr2=0x7fd67c0f6bd0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:35:06.079 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:51.685+0000 7fd681daa700 1 --2- 172.21.15.66:0/2923212451 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fd67c060220 0x7fd67c0f6bd0 secure :-1 s=READY pgs=428 cs=0 l=1 rev1=1 rx=0x7fd66c013510 tx=0x7fd66c00b040).stop 2023-11-17T05:35:06.079 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:51.685+0000 7fd681daa700 1 -- 172.21.15.66:0/2923212451 shutdown_connections 2023-11-17T05:35:06.079 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:51.685+0000 7fd681daa700 1 --2- 172.21.15.66:0/2923212451 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fd66402ff60 0x7fd664032420 unknown :-1 s=CLOSED pgs=179 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:35:06.079 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:51.685+0000 7fd681daa700 1 --2- 172.21.15.66:0/2923212451 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fd67c060220 0x7fd67c0f6bd0 unknown :-1 s=CLOSED pgs=428 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:35:06.079 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:51.685+0000 7fd681daa700 1 -- 172.21.15.66:0/2923212451 >> 172.21.15.66:0/2923212451 conn(0x7fd67c0f0b70 msgr2=0x7fd67c0f1660 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:35:06.079 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:51.685+0000 7fd681daa700 1 -- 172.21.15.66:0/2923212451 shutdown_connections 2023-11-17T05:35:06.079 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:34:51.685+0000 7fd681daa700 1 -- 172.21.15.66:0/2923212451 wait complete. 2023-11-17T05:35:06.079 INFO:teuthology.orchestra.run.smithi066.stderr:dumped monmap epoch 1 2023-11-17T05:35:07.081 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2023-11-17T05:35:07.081 DEBUG:teuthology.orchestra.run.smithi066:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.4 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ed6470e2-8508-11ee-8dbe-212e2dc638e7 -- ceph mon dump -f json 2023-11-17T05:35:07.989 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:35:07 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:35:07.552860+0000 mgr.smithi040.ogcprl (mgr.14164) 429 : cluster [DBG] pgmap v413: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:35:08.277 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:08.276+0000 7eff625f9700 1 -- 172.21.15.66:0/1819218931 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7eff5c0f5410 msgr2=0x7eff5c0f5830 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:35:08.278 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:08.276+0000 7eff625f9700 1 --2- 172.21.15.66:0/1819218931 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7eff5c0f5410 0x7eff5c0f5830 secure :-1 s=READY pgs=429 cs=0 l=1 rev1=1 rx=0x7eff4c009fb0 tx=0x7eff4c009b30).stop 2023-11-17T05:35:08.278 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:08.276+0000 7eff625f9700 1 -- 172.21.15.66:0/1819218931 shutdown_connections 2023-11-17T05:35:08.278 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:08.276+0000 7eff625f9700 1 --2- 172.21.15.66:0/1819218931 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7eff5c0f5410 0x7eff5c0f5830 unknown :-1 s=CLOSED pgs=429 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:35:08.278 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:08.276+0000 7eff625f9700 1 -- 172.21.15.66:0/1819218931 >> 172.21.15.66:0/1819218931 conn(0x7eff5c0f0b70 msgr2=0x7eff5c0f2f90 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:35:08.278 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:08.276+0000 7eff625f9700 1 -- 172.21.15.66:0/1819218931 shutdown_connections 2023-11-17T05:35:08.278 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:08.276+0000 7eff625f9700 1 -- 172.21.15.66:0/1819218931 wait complete. 2023-11-17T05:35:08.279 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:08.277+0000 7eff625f9700 1 Processor -- start 2023-11-17T05:35:08.279 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:08.277+0000 7eff625f9700 1 -- start start 2023-11-17T05:35:08.279 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:08.277+0000 7eff625f9700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7eff5c0f5410 0x7eff5c109450 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:35:08.279 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:08.277+0000 7eff625f9700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7eff5c060220 con 0x7eff5c0f5410 2023-11-17T05:35:08.279 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:08.277+0000 7eff5bfff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7eff5c0f5410 0x7eff5c109450 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:35:08.279 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:08.277+0000 7eff5bfff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7eff5c0f5410 0x7eff5c109450 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.66:59632/0 (socket says 172.21.15.66:59632) 2023-11-17T05:35:08.280 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:08.277+0000 7eff5bfff700 1 -- 172.21.15.66:0/1982308679 learned_addr learned my addr 172.21.15.66:0/1982308679 (peer_addr_for_me v2:172.21.15.66:0/0) 2023-11-17T05:35:08.280 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:08.278+0000 7eff5bfff700 1 -- 172.21.15.66:0/1982308679 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7eff4c005040 con 0x7eff5c0f5410 2023-11-17T05:35:08.280 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:08.278+0000 7eff5bfff700 1 --2- 172.21.15.66:0/1982308679 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7eff5c0f5410 0x7eff5c109450 secure :-1 s=READY pgs=430 cs=0 l=1 rev1=1 rx=0x7eff4c013510 tx=0x7eff4c00b040).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:35:08.280 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:08.278+0000 7eff58ff9700 1 -- 172.21.15.66:0/1982308679 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7eff4c009320 con 0x7eff5c0f5410 2023-11-17T05:35:08.280 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:08.278+0000 7eff625f9700 1 -- 172.21.15.66:0/1982308679 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7eff5c060420 con 0x7eff5c0f5410 2023-11-17T05:35:08.280 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:08.278+0000 7eff58ff9700 1 -- 172.21.15.66:0/1982308679 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(25 keys) v1 ==== 1074+0+0 (secure 0 0 0) 0x7eff4c013f00 con 0x7eff5c0f5410 2023-11-17T05:35:08.280 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:08.278+0000 7eff58ff9700 1 -- 172.21.15.66:0/1982308679 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7eff4c014d90 con 0x7eff5c0f5410 2023-11-17T05:35:08.280 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:08.278+0000 7eff625f9700 1 -- 172.21.15.66:0/1982308679 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7eff5c109b00 con 0x7eff5c0f5410 2023-11-17T05:35:08.281 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:08.279+0000 7eff58ff9700 1 -- 172.21.15.66:0/1982308679 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 17) v1 ==== 36269+0+0 (secure 0 0 0) 0x7eff4c0095f0 con 0x7eff5c0f5410 2023-11-17T05:35:08.281 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:08.279+0000 7eff625f9700 1 -- 172.21.15.66:0/1982308679 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7eff5c059100 con 0x7eff5c0f5410 2023-11-17T05:35:08.281 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:08.279+0000 7eff58ff9700 1 --2- 172.21.15.66:0/1982308679 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7eff4402ff60 0x7eff44032420 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:35:08.281 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:08.279+0000 7eff58ff9700 1 -- 172.21.15.66:0/1982308679 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(5..5 src has 1..5) v4 ==== 1270+0+0 (secure 0 0 0) 0x7eff4c041080 con 0x7eff5c0f5410 2023-11-17T05:35:08.281 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:08.279+0000 7eff5b7fe700 1 --2- 172.21.15.66:0/1982308679 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7eff4402ff60 0x7eff44032420 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:35:08.281 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:08.280+0000 7eff5b7fe700 1 --2- 172.21.15.66:0/1982308679 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7eff4402ff60 0x7eff44032420 secure :-1 s=READY pgs=180 cs=0 l=1 rev1=1 rx=0x7eff500096a0 tx=0x7eff50006b40).ready entity=mgr.14164 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:35:08.285 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:08.284+0000 7eff58ff9700 1 -- 172.21.15.66:0/1982308679 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+152212 (secure 0 0 0) 0x7eff4c014420 con 0x7eff5c0f5410 2023-11-17T05:35:08.505 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:08.503+0000 7eff625f9700 1 -- 172.21.15.66:0/1982308679 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "mon dump", "format": "json"} v 0) v1 -- 0x7eff5c045ad0 con 0x7eff5c0f5410 2023-11-17T05:35:08.505 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:08.504+0000 7eff58ff9700 1 -- 172.21.15.66:0/1982308679 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mon_command_ack([{"prefix": "mon dump", "format": "json"}]=0 dumped monmap epoch 1 v1) v1 ==== 95+0+690 (secure 0 0 0) 0x7eff4c01d7f0 con 0x7eff5c0f5410 2023-11-17T05:35:08.505 INFO:teuthology.orchestra.run.smithi066.stdout: 2023-11-17T05:35:08.506 INFO:teuthology.orchestra.run.smithi066.stdout:{"epoch":1,"fsid":"ed6470e2-8508-11ee-8dbe-212e2dc638e7","modified":"2023-11-17T05:20:13.934038Z","created":"2023-11-17T05:20:13.934038Z","min_mon_release":16,"min_mon_release_name":"pacific","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging"],"optional":[]},"mons":[{"rank":0,"name":"smithi040","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:3300","nonce":0},{"type":"v1","addr":"172.21.15.40:6789","nonce":0}]},"addr":"172.21.15.40:6789/0","public_addr":"172.21.15.40:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2023-11-17T05:35:08.507 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:08.506+0000 7eff625f9700 1 -- 172.21.15.66:0/1982308679 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7eff4402ff60 msgr2=0x7eff44032420 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:35:08.507 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:08.506+0000 7eff625f9700 1 --2- 172.21.15.66:0/1982308679 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7eff4402ff60 0x7eff44032420 secure :-1 s=READY pgs=180 cs=0 l=1 rev1=1 rx=0x7eff500096a0 tx=0x7eff50006b40).stop 2023-11-17T05:35:08.508 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:08.506+0000 7eff625f9700 1 -- 172.21.15.66:0/1982308679 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7eff5c0f5410 msgr2=0x7eff5c109450 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:35:08.508 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:08.506+0000 7eff625f9700 1 --2- 172.21.15.66:0/1982308679 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7eff5c0f5410 0x7eff5c109450 secure :-1 s=READY pgs=430 cs=0 l=1 rev1=1 rx=0x7eff4c013510 tx=0x7eff4c00b040).stop 2023-11-17T05:35:08.508 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:08.506+0000 7eff625f9700 1 -- 172.21.15.66:0/1982308679 shutdown_connections 2023-11-17T05:35:08.508 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:08.506+0000 7eff625f9700 1 --2- 172.21.15.66:0/1982308679 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7eff4402ff60 0x7eff44032420 unknown :-1 s=CLOSED pgs=180 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:35:08.508 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:08.506+0000 7eff625f9700 1 --2- 172.21.15.66:0/1982308679 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7eff5c0f5410 0x7eff5c109450 unknown :-1 s=CLOSED pgs=430 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:35:08.508 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:08.506+0000 7eff625f9700 1 -- 172.21.15.66:0/1982308679 >> 172.21.15.66:0/1982308679 conn(0x7eff5c0f0b70 msgr2=0x7eff5c0f1660 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:35:08.508 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:08.506+0000 7eff625f9700 1 -- 172.21.15.66:0/1982308679 shutdown_connections 2023-11-17T05:35:08.508 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:08.506+0000 7eff625f9700 1 -- 172.21.15.66:0/1982308679 wait complete. 2023-11-17T05:35:08.508 INFO:teuthology.orchestra.run.smithi066.stderr:dumped monmap epoch 1 2023-11-17T05:35:08.988 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:35:08 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:35:08.505924+0000 mon.smithi040 (mon.0) 331 : audit [DBG] from='client.? 172.21.15.66:0/1982308679' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-11-17T05:35:09.945 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2023-11-17T05:35:09.945 DEBUG:teuthology.orchestra.run.smithi066:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.4 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ed6470e2-8508-11ee-8dbe-212e2dc638e7 -- ceph mon dump -f json 2023-11-17T05:35:10.989 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:35:10 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:35:09.553126+0000 mgr.smithi040.ogcprl (mgr.14164) 430 : cluster [DBG] pgmap v414: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:35:11.092 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:11.089+0000 7f7dc2be6700 1 -- 172.21.15.66:0/2192186694 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f7dbc0f51a0 msgr2=0x7f7dbc0f55c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:35:11.092 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:11.089+0000 7f7dc2be6700 1 --2- 172.21.15.66:0/2192186694 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f7dbc0f51a0 0x7f7dbc0f55c0 secure :-1 s=READY pgs=431 cs=0 l=1 rev1=1 rx=0x7f7db8009fb0 tx=0x7f7db8009b30).stop 2023-11-17T05:35:11.092 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:11.089+0000 7f7dc2be6700 1 -- 172.21.15.66:0/2192186694 shutdown_connections 2023-11-17T05:35:11.092 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:11.089+0000 7f7dc2be6700 1 --2- 172.21.15.66:0/2192186694 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f7dbc0f51a0 0x7f7dbc0f55c0 unknown :-1 s=CLOSED pgs=431 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:35:11.092 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:11.089+0000 7f7dc2be6700 1 -- 172.21.15.66:0/2192186694 >> 172.21.15.66:0/2192186694 conn(0x7f7dbc0f09a0 msgr2=0x7f7dbc0f2dc0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:35:11.092 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:11.089+0000 7f7dc2be6700 1 -- 172.21.15.66:0/2192186694 shutdown_connections 2023-11-17T05:35:11.092 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:11.089+0000 7f7dc2be6700 1 -- 172.21.15.66:0/2192186694 wait complete. 2023-11-17T05:35:11.092 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:11.089+0000 7f7dc2be6700 1 Processor -- start 2023-11-17T05:35:11.093 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:11.089+0000 7f7dc2be6700 1 -- start start 2023-11-17T05:35:11.093 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:11.090+0000 7f7dc2be6700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f7dbc0f51a0 0x7f7dbc1064b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:35:11.093 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:11.090+0000 7f7dc2be6700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f7dbc1069a0 con 0x7f7dbc0f51a0 2023-11-17T05:35:11.093 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:11.090+0000 7f7dc1be4700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f7dbc0f51a0 0x7f7dbc1064b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:35:11.093 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:11.090+0000 7f7dc1be4700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f7dbc0f51a0 0x7f7dbc1064b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.66:59640/0 (socket says 172.21.15.66:59640) 2023-11-17T05:35:11.093 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:11.090+0000 7f7dc1be4700 1 -- 172.21.15.66:0/1450585467 learned_addr learned my addr 172.21.15.66:0/1450585467 (peer_addr_for_me v2:172.21.15.66:0/0) 2023-11-17T05:35:11.093 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:11.090+0000 7f7dc1be4700 1 -- 172.21.15.66:0/1450585467 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f7db8005040 con 0x7f7dbc0f51a0 2023-11-17T05:35:11.093 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:11.090+0000 7f7dc1be4700 1 --2- 172.21.15.66:0/1450585467 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f7dbc0f51a0 0x7f7dbc1064b0 secure :-1 s=READY pgs=432 cs=0 l=1 rev1=1 rx=0x7f7db8013510 tx=0x7f7db800b040).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:35:11.093 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:11.090+0000 7f7db27fc700 1 -- 172.21.15.66:0/1450585467 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f7db8009320 con 0x7f7dbc0f51a0 2023-11-17T05:35:11.093 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:11.090+0000 7f7dc2be6700 1 -- 172.21.15.66:0/1450585467 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f7dbc106ba0 con 0x7f7dbc0f51a0 2023-11-17T05:35:11.093 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:11.090+0000 7f7db27fc700 1 -- 172.21.15.66:0/1450585467 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(25 keys) v1 ==== 1074+0+0 (secure 0 0 0) 0x7f7db8013f00 con 0x7f7dbc0f51a0 2023-11-17T05:35:11.093 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:11.090+0000 7f7db27fc700 1 -- 172.21.15.66:0/1450585467 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f7db8014dc0 con 0x7f7dbc0f51a0 2023-11-17T05:35:11.094 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:11.091+0000 7f7dc2be6700 1 -- 172.21.15.66:0/1450585467 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f7dbc107030 con 0x7f7dbc0f51a0 2023-11-17T05:35:11.094 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:11.091+0000 7f7db27fc700 1 -- 172.21.15.66:0/1450585467 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 17) v1 ==== 36269+0+0 (secure 0 0 0) 0x7f7db80095f0 con 0x7f7dbc0f51a0 2023-11-17T05:35:11.095 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:11.091+0000 7f7dc2be6700 1 -- 172.21.15.66:0/1450585467 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f7dbc045ad0 con 0x7f7dbc0f51a0 2023-11-17T05:35:11.095 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:11.092+0000 7f7db27fc700 1 --2- 172.21.15.66:0/1450585467 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f7da802ff60 0x7f7da8032420 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:35:11.095 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:11.092+0000 7f7db27fc700 1 -- 172.21.15.66:0/1450585467 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(5..5 src has 1..5) v4 ==== 1270+0+0 (secure 0 0 0) 0x7f7db8041000 con 0x7f7dbc0f51a0 2023-11-17T05:35:11.095 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:11.092+0000 7f7dc13e3700 1 --2- 172.21.15.66:0/1450585467 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f7da802ff60 0x7f7da8032420 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:35:11.095 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:11.092+0000 7f7dc13e3700 1 --2- 172.21.15.66:0/1450585467 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f7da802ff60 0x7f7da8032420 secure :-1 s=READY pgs=181 cs=0 l=1 rev1=1 rx=0x7f7dac0096a0 tx=0x7f7dac006b40).ready entity=mgr.14164 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:35:11.097 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:11.096+0000 7f7db27fc700 1 -- 172.21.15.66:0/1450585467 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+152212 (secure 0 0 0) 0x7f7db8014420 con 0x7f7dbc0f51a0 2023-11-17T05:35:11.318 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:11.316+0000 7f7dc2be6700 1 -- 172.21.15.66:0/1450585467 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "mon dump", "format": "json"} v 0) v1 -- 0x7f7dbc107ee0 con 0x7f7dbc0f51a0 2023-11-17T05:35:11.318 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:11.317+0000 7f7db27fc700 1 -- 172.21.15.66:0/1450585467 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mon_command_ack([{"prefix": "mon dump", "format": "json"}]=0 dumped monmap epoch 1 v1) v1 ==== 95+0+690 (secure 0 0 0) 0x7f7db8010230 con 0x7f7dbc0f51a0 2023-11-17T05:35:11.319 INFO:teuthology.orchestra.run.smithi066.stdout: 2023-11-17T05:35:11.319 INFO:teuthology.orchestra.run.smithi066.stdout:{"epoch":1,"fsid":"ed6470e2-8508-11ee-8dbe-212e2dc638e7","modified":"2023-11-17T05:20:13.934038Z","created":"2023-11-17T05:20:13.934038Z","min_mon_release":16,"min_mon_release_name":"pacific","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging"],"optional":[]},"mons":[{"rank":0,"name":"smithi040","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:3300","nonce":0},{"type":"v1","addr":"172.21.15.40:6789","nonce":0}]},"addr":"172.21.15.40:6789/0","public_addr":"172.21.15.40:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2023-11-17T05:35:11.321 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:11.318+0000 7f7dc2be6700 1 -- 172.21.15.66:0/1450585467 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f7da802ff60 msgr2=0x7f7da8032420 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:35:11.321 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:11.318+0000 7f7dc2be6700 1 --2- 172.21.15.66:0/1450585467 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f7da802ff60 0x7f7da8032420 secure :-1 s=READY pgs=181 cs=0 l=1 rev1=1 rx=0x7f7dac0096a0 tx=0x7f7dac006b40).stop 2023-11-17T05:35:11.321 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:11.318+0000 7f7dc2be6700 1 -- 172.21.15.66:0/1450585467 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f7dbc0f51a0 msgr2=0x7f7dbc1064b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:35:11.321 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:11.318+0000 7f7dc2be6700 1 --2- 172.21.15.66:0/1450585467 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f7dbc0f51a0 0x7f7dbc1064b0 secure :-1 s=READY pgs=432 cs=0 l=1 rev1=1 rx=0x7f7db8013510 tx=0x7f7db800b040).stop 2023-11-17T05:35:11.322 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:11.318+0000 7f7dc2be6700 1 -- 172.21.15.66:0/1450585467 shutdown_connections 2023-11-17T05:35:11.322 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:11.318+0000 7f7dc2be6700 1 --2- 172.21.15.66:0/1450585467 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f7da802ff60 0x7f7da8032420 unknown :-1 s=CLOSED pgs=181 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:35:11.322 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:11.318+0000 7f7dc2be6700 1 --2- 172.21.15.66:0/1450585467 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f7dbc0f51a0 0x7f7dbc1064b0 unknown :-1 s=CLOSED pgs=432 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:35:11.322 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:11.319+0000 7f7dc2be6700 1 -- 172.21.15.66:0/1450585467 >> 172.21.15.66:0/1450585467 conn(0x7f7dbc0f09a0 msgr2=0x7f7dbc0f1400 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:35:11.322 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:11.319+0000 7f7dc2be6700 1 -- 172.21.15.66:0/1450585467 shutdown_connections 2023-11-17T05:35:11.322 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:11.319+0000 7f7dc2be6700 1 -- 172.21.15.66:0/1450585467 wait complete. 2023-11-17T05:35:11.322 INFO:teuthology.orchestra.run.smithi066.stderr:dumped monmap epoch 1 2023-11-17T05:35:11.989 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:35:11 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:35:11.318751+0000 mon.smithi040 (mon.0) 332 : audit [DBG] from='client.? 172.21.15.66:0/1450585467' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-11-17T05:35:11.989 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:35:11 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:35:11.323567+0000 mon.smithi040 (mon.0) 333 : audit [INF] from='mgr.14164 172.21.15.40:0/2203032451' entity='mgr.smithi040.ogcprl' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi040.ogcprl/mirror_snapshot_schedule"}]: dispatch 2023-11-17T05:35:11.989 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:35:11 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:35:11.323793+0000 mon.smithi040 (mon.0) 334 : audit [INF] from='mgr.14164 172.21.15.40:0/2203032451' entity='mgr.smithi040.ogcprl' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi040.ogcprl/trash_purge_schedule"}]: dispatch 2023-11-17T05:35:12.703 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2023-11-17T05:35:12.704 DEBUG:teuthology.orchestra.run.smithi066:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.4 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ed6470e2-8508-11ee-8dbe-212e2dc638e7 -- ceph mon dump -f json 2023-11-17T05:35:12.989 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:35:12 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:35:11.553503+0000 mgr.smithi040.ogcprl (mgr.14164) 431 : cluster [DBG] pgmap v415: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:35:13.863 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:13.858+0000 7f9ec0831700 1 -- 172.21.15.66:0/1920685702 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f9ebc0f5410 msgr2=0x7f9ebc0f5830 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:35:13.863 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:13.858+0000 7f9ec0831700 1 --2- 172.21.15.66:0/1920685702 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f9ebc0f5410 0x7f9ebc0f5830 secure :-1 s=READY pgs=433 cs=0 l=1 rev1=1 rx=0x7f9ea4009fb0 tx=0x7f9ea4009b30).stop 2023-11-17T05:35:13.863 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:13.858+0000 7f9ec0831700 1 -- 172.21.15.66:0/1920685702 shutdown_connections 2023-11-17T05:35:13.864 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:13.858+0000 7f9ec0831700 1 --2- 172.21.15.66:0/1920685702 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f9ebc0f5410 0x7f9ebc0f5830 unknown :-1 s=CLOSED pgs=433 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:35:13.864 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:13.858+0000 7f9ec0831700 1 -- 172.21.15.66:0/1920685702 >> 172.21.15.66:0/1920685702 conn(0x7f9ebc0f0b70 msgr2=0x7f9ebc0f2f90 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:35:13.864 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:13.858+0000 7f9ec0831700 1 -- 172.21.15.66:0/1920685702 shutdown_connections 2023-11-17T05:35:13.864 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:13.858+0000 7f9ec0831700 1 -- 172.21.15.66:0/1920685702 wait complete. 2023-11-17T05:35:13.864 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:13.859+0000 7f9ec0831700 1 Processor -- start 2023-11-17T05:35:13.864 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:13.859+0000 7f9ec0831700 1 -- start start 2023-11-17T05:35:13.864 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:13.859+0000 7f9ec0831700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f9ebc0f5410 0x7f9ebc109450 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:35:13.864 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:13.859+0000 7f9ec0831700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f9ebc060220 con 0x7f9ebc0f5410 2023-11-17T05:35:13.865 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:13.859+0000 7f9eba59c700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f9ebc0f5410 0x7f9ebc109450 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:35:13.865 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:13.859+0000 7f9eba59c700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f9ebc0f5410 0x7f9ebc109450 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.66:54632/0 (socket says 172.21.15.66:54632) 2023-11-17T05:35:13.865 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:13.859+0000 7f9eba59c700 1 -- 172.21.15.66:0/298608372 learned_addr learned my addr 172.21.15.66:0/298608372 (peer_addr_for_me v2:172.21.15.66:0/0) 2023-11-17T05:35:13.865 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:13.860+0000 7f9eba59c700 1 -- 172.21.15.66:0/298608372 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f9ea4005040 con 0x7f9ebc0f5410 2023-11-17T05:35:13.865 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:13.860+0000 7f9eba59c700 1 --2- 172.21.15.66:0/298608372 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f9ebc0f5410 0x7f9ebc109450 secure :-1 s=READY pgs=434 cs=0 l=1 rev1=1 rx=0x7f9ea4013510 tx=0x7f9ea400b040).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:35:13.865 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:13.860+0000 7f9eb2ffd700 1 -- 172.21.15.66:0/298608372 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f9ea4009320 con 0x7f9ebc0f5410 2023-11-17T05:35:13.866 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:13.860+0000 7f9ec0831700 1 -- 172.21.15.66:0/298608372 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f9ebc060420 con 0x7f9ebc0f5410 2023-11-17T05:35:13.866 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:13.860+0000 7f9eb2ffd700 1 -- 172.21.15.66:0/298608372 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(25 keys) v1 ==== 1074+0+0 (secure 0 0 0) 0x7f9ea4013f00 con 0x7f9ebc0f5410 2023-11-17T05:35:13.866 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:13.860+0000 7f9eb2ffd700 1 -- 172.21.15.66:0/298608372 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f9ea4014d90 con 0x7f9ebc0f5410 2023-11-17T05:35:13.866 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:13.860+0000 7f9ec0831700 1 -- 172.21.15.66:0/298608372 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f9ebc109b00 con 0x7f9ebc0f5410 2023-11-17T05:35:13.867 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:13.861+0000 7f9eb2ffd700 1 -- 172.21.15.66:0/298608372 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 17) v1 ==== 36269+0+0 (secure 0 0 0) 0x7f9ea40095f0 con 0x7f9ebc0f5410 2023-11-17T05:35:13.867 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:13.861+0000 7f9ec0831700 1 -- 172.21.15.66:0/298608372 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f9ebc059100 con 0x7f9ebc0f5410 2023-11-17T05:35:13.880 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:13.861+0000 7f9eb2ffd700 1 --2- 172.21.15.66:0/298608372 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f9ea802ff60 0x7f9ea8032420 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:35:13.880 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:13.861+0000 7f9eb2ffd700 1 -- 172.21.15.66:0/298608372 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(5..5 src has 1..5) v4 ==== 1270+0+0 (secure 0 0 0) 0x7f9ea4041000 con 0x7f9ebc0f5410 2023-11-17T05:35:13.881 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:13.861+0000 7f9eb9d9b700 1 --2- 172.21.15.66:0/298608372 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f9ea802ff60 0x7f9ea8032420 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:35:13.881 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:13.862+0000 7f9eb9d9b700 1 --2- 172.21.15.66:0/298608372 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f9ea802ff60 0x7f9ea8032420 secure :-1 s=READY pgs=182 cs=0 l=1 rev1=1 rx=0x7f9eac0096a0 tx=0x7f9eac006b40).ready entity=mgr.14164 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:35:13.884 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:13.866+0000 7f9eb2ffd700 1 -- 172.21.15.66:0/298608372 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+152212 (secure 0 0 0) 0x7f9ea4014420 con 0x7f9ebc0f5410 2023-11-17T05:35:13.989 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:35:13 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:35:13.553890+0000 mgr.smithi040.ogcprl (mgr.14164) 432 : cluster [DBG] pgmap v416: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:35:14.085 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:14.084+0000 7f9ec0831700 1 -- 172.21.15.66:0/298608372 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "mon dump", "format": "json"} v 0) v1 -- 0x7f9ebc045ad0 con 0x7f9ebc0f5410 2023-11-17T05:35:14.086 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:14.085+0000 7f9eb2ffd700 1 -- 172.21.15.66:0/298608372 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mon_command_ack([{"prefix": "mon dump", "format": "json"}]=0 dumped monmap epoch 1 v1) v1 ==== 95+0+690 (secure 0 0 0) 0x7f9ea4010230 con 0x7f9ebc0f5410 2023-11-17T05:35:14.086 INFO:teuthology.orchestra.run.smithi066.stdout: 2023-11-17T05:35:14.086 INFO:teuthology.orchestra.run.smithi066.stdout:{"epoch":1,"fsid":"ed6470e2-8508-11ee-8dbe-212e2dc638e7","modified":"2023-11-17T05:20:13.934038Z","created":"2023-11-17T05:20:13.934038Z","min_mon_release":16,"min_mon_release_name":"pacific","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging"],"optional":[]},"mons":[{"rank":0,"name":"smithi040","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:3300","nonce":0},{"type":"v1","addr":"172.21.15.40:6789","nonce":0}]},"addr":"172.21.15.40:6789/0","public_addr":"172.21.15.40:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2023-11-17T05:35:14.087 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:14.086+0000 7f9ec0831700 1 -- 172.21.15.66:0/298608372 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f9ea802ff60 msgr2=0x7f9ea8032420 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:35:14.087 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:14.086+0000 7f9ec0831700 1 --2- 172.21.15.66:0/298608372 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f9ea802ff60 0x7f9ea8032420 secure :-1 s=READY pgs=182 cs=0 l=1 rev1=1 rx=0x7f9eac0096a0 tx=0x7f9eac006b40).stop 2023-11-17T05:35:14.088 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:14.086+0000 7f9ec0831700 1 -- 172.21.15.66:0/298608372 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f9ebc0f5410 msgr2=0x7f9ebc109450 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:35:14.088 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:14.086+0000 7f9ec0831700 1 --2- 172.21.15.66:0/298608372 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f9ebc0f5410 0x7f9ebc109450 secure :-1 s=READY pgs=434 cs=0 l=1 rev1=1 rx=0x7f9ea4013510 tx=0x7f9ea400b040).stop 2023-11-17T05:35:14.088 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:14.086+0000 7f9ec0831700 1 -- 172.21.15.66:0/298608372 shutdown_connections 2023-11-17T05:35:14.088 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:14.086+0000 7f9ec0831700 1 --2- 172.21.15.66:0/298608372 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f9ea802ff60 0x7f9ea8032420 unknown :-1 s=CLOSED pgs=182 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:35:14.088 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:14.086+0000 7f9ec0831700 1 --2- 172.21.15.66:0/298608372 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f9ebc0f5410 0x7f9ebc109450 unknown :-1 s=CLOSED pgs=434 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:35:14.088 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:14.086+0000 7f9ec0831700 1 -- 172.21.15.66:0/298608372 >> 172.21.15.66:0/298608372 conn(0x7f9ebc0f0b70 msgr2=0x7f9ebc0f1660 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:35:14.088 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:14.086+0000 7f9ec0831700 1 -- 172.21.15.66:0/298608372 shutdown_connections 2023-11-17T05:35:14.088 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:14.086+0000 7f9ec0831700 1 -- 172.21.15.66:0/298608372 wait complete. 2023-11-17T05:35:14.088 INFO:teuthology.orchestra.run.smithi066.stderr:dumped monmap epoch 1 2023-11-17T05:35:14.989 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:35:14 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:35:14.086146+0000 mon.smithi040 (mon.0) 335 : audit [DBG] from='client.? 172.21.15.66:0/298608372' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-11-17T05:35:15.467 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2023-11-17T05:35:15.467 DEBUG:teuthology.orchestra.run.smithi066:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.4 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ed6470e2-8508-11ee-8dbe-212e2dc638e7 -- ceph mon dump -f json 2023-11-17T05:35:15.988 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:35:15 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:35:15.554176+0000 mgr.smithi040.ogcprl (mgr.14164) 433 : cluster [DBG] pgmap v417: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:35:16.669 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:16.666+0000 7f18fa5a5700 1 -- 172.21.15.66:0/2836028032 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f18f40f5410 msgr2=0x7f18f40f5830 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:35:16.669 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:16.666+0000 7f18fa5a5700 1 --2- 172.21.15.66:0/2836028032 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f18f40f5410 0x7f18f40f5830 secure :-1 s=READY pgs=435 cs=0 l=1 rev1=1 rx=0x7f18e4009fb0 tx=0x7f18e4009b30).stop 2023-11-17T05:35:16.669 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:16.666+0000 7f18fa5a5700 1 -- 172.21.15.66:0/2836028032 shutdown_connections 2023-11-17T05:35:16.669 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:16.666+0000 7f18fa5a5700 1 --2- 172.21.15.66:0/2836028032 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f18f40f5410 0x7f18f40f5830 unknown :-1 s=CLOSED pgs=435 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:35:16.670 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:16.666+0000 7f18fa5a5700 1 -- 172.21.15.66:0/2836028032 >> 172.21.15.66:0/2836028032 conn(0x7f18f40f0b70 msgr2=0x7f18f40f2f90 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:35:16.670 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:16.666+0000 7f18fa5a5700 1 -- 172.21.15.66:0/2836028032 shutdown_connections 2023-11-17T05:35:16.670 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:16.666+0000 7f18fa5a5700 1 -- 172.21.15.66:0/2836028032 wait complete. 2023-11-17T05:35:16.670 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:16.667+0000 7f18fa5a5700 1 Processor -- start 2023-11-17T05:35:16.670 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:16.667+0000 7f18fa5a5700 1 -- start start 2023-11-17T05:35:16.670 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:16.667+0000 7f18fa5a5700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f18f40f5410 0x7f18f40fd3a0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:35:16.670 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:16.667+0000 7f18fa5a5700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f18f40fb820 con 0x7f18f40f5410 2023-11-17T05:35:16.670 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:16.667+0000 7f18f3fff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f18f40f5410 0x7f18f40fd3a0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:35:16.670 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:16.667+0000 7f18f3fff700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f18f40f5410 0x7f18f40fd3a0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.66:54650/0 (socket says 172.21.15.66:54650) 2023-11-17T05:35:16.670 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:16.667+0000 7f18f3fff700 1 -- 172.21.15.66:0/3354041880 learned_addr learned my addr 172.21.15.66:0/3354041880 (peer_addr_for_me v2:172.21.15.66:0/0) 2023-11-17T05:35:16.670 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:16.668+0000 7f18f3fff700 1 -- 172.21.15.66:0/3354041880 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f18e4005040 con 0x7f18f40f5410 2023-11-17T05:35:16.670 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:16.668+0000 7f18f3fff700 1 --2- 172.21.15.66:0/3354041880 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f18f40f5410 0x7f18f40fd3a0 secure :-1 s=READY pgs=436 cs=0 l=1 rev1=1 rx=0x7f18e4013510 tx=0x7f18e400b040).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:35:16.671 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:16.668+0000 7f18f0ff9700 1 -- 172.21.15.66:0/3354041880 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f18e4009320 con 0x7f18f40f5410 2023-11-17T05:35:16.671 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:16.668+0000 7f18fa5a5700 1 -- 172.21.15.66:0/3354041880 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f18f40fb9c0 con 0x7f18f40f5410 2023-11-17T05:35:16.671 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:16.668+0000 7f18f0ff9700 1 -- 172.21.15.66:0/3354041880 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(25 keys) v1 ==== 1074+0+0 (secure 0 0 0) 0x7f18e4013f00 con 0x7f18f40f5410 2023-11-17T05:35:16.671 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:16.668+0000 7f18f0ff9700 1 -- 172.21.15.66:0/3354041880 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f18e4014d90 con 0x7f18f40f5410 2023-11-17T05:35:16.671 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:16.668+0000 7f18fa5a5700 1 -- 172.21.15.66:0/3354041880 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f18f40fbe50 con 0x7f18f40f5410 2023-11-17T05:35:16.672 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:16.669+0000 7f18f0ff9700 1 -- 172.21.15.66:0/3354041880 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 17) v1 ==== 36269+0+0 (secure 0 0 0) 0x7f18e40095f0 con 0x7f18f40f5410 2023-11-17T05:35:16.672 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:16.669+0000 7f18fa5a5700 1 -- 172.21.15.66:0/3354041880 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f18f4059100 con 0x7f18f40f5410 2023-11-17T05:35:16.672 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:16.669+0000 7f18f0ff9700 1 --2- 172.21.15.66:0/3354041880 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f18dc02ff60 0x7f18dc032420 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:35:16.672 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:16.669+0000 7f18f0ff9700 1 -- 172.21.15.66:0/3354041880 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(5..5 src has 1..5) v4 ==== 1270+0+0 (secure 0 0 0) 0x7f18e4040fc0 con 0x7f18f40f5410 2023-11-17T05:35:16.672 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:16.669+0000 7f18f37fe700 1 --2- 172.21.15.66:0/3354041880 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f18dc02ff60 0x7f18dc032420 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:35:16.672 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:16.670+0000 7f18f37fe700 1 --2- 172.21.15.66:0/3354041880 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f18dc02ff60 0x7f18dc032420 secure :-1 s=READY pgs=183 cs=0 l=1 rev1=1 rx=0x7f18e80096a0 tx=0x7f18e8006b40).ready entity=mgr.14164 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:35:16.675 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:16.674+0000 7f18f0ff9700 1 -- 172.21.15.66:0/3354041880 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+152212 (secure 0 0 0) 0x7f18e4014420 con 0x7f18f40f5410 2023-11-17T05:35:16.894 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:16.893+0000 7f18fa5a5700 1 -- 172.21.15.66:0/3354041880 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "mon dump", "format": "json"} v 0) v1 -- 0x7f18f4045ad0 con 0x7f18f40f5410 2023-11-17T05:35:16.895 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:16.894+0000 7f18f0ff9700 1 -- 172.21.15.66:0/3354041880 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mon_command_ack([{"prefix": "mon dump", "format": "json"}]=0 dumped monmap epoch 1 v1) v1 ==== 95+0+690 (secure 0 0 0) 0x7f18e4010230 con 0x7f18f40f5410 2023-11-17T05:35:16.895 INFO:teuthology.orchestra.run.smithi066.stdout: 2023-11-17T05:35:16.895 INFO:teuthology.orchestra.run.smithi066.stdout:{"epoch":1,"fsid":"ed6470e2-8508-11ee-8dbe-212e2dc638e7","modified":"2023-11-17T05:20:13.934038Z","created":"2023-11-17T05:20:13.934038Z","min_mon_release":16,"min_mon_release_name":"pacific","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging"],"optional":[]},"mons":[{"rank":0,"name":"smithi040","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:3300","nonce":0},{"type":"v1","addr":"172.21.15.40:6789","nonce":0}]},"addr":"172.21.15.40:6789/0","public_addr":"172.21.15.40:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2023-11-17T05:35:16.897 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:16.895+0000 7f18fa5a5700 1 -- 172.21.15.66:0/3354041880 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f18dc02ff60 msgr2=0x7f18dc032420 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:35:16.897 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:16.895+0000 7f18fa5a5700 1 --2- 172.21.15.66:0/3354041880 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f18dc02ff60 0x7f18dc032420 secure :-1 s=READY pgs=183 cs=0 l=1 rev1=1 rx=0x7f18e80096a0 tx=0x7f18e8006b40).stop 2023-11-17T05:35:16.897 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:16.895+0000 7f18fa5a5700 1 -- 172.21.15.66:0/3354041880 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f18f40f5410 msgr2=0x7f18f40fd3a0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:35:16.897 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:16.895+0000 7f18fa5a5700 1 --2- 172.21.15.66:0/3354041880 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f18f40f5410 0x7f18f40fd3a0 secure :-1 s=READY pgs=436 cs=0 l=1 rev1=1 rx=0x7f18e4013510 tx=0x7f18e400b040).stop 2023-11-17T05:35:16.897 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:16.895+0000 7f18fa5a5700 1 -- 172.21.15.66:0/3354041880 shutdown_connections 2023-11-17T05:35:16.897 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:16.895+0000 7f18fa5a5700 1 --2- 172.21.15.66:0/3354041880 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f18dc02ff60 0x7f18dc032420 unknown :-1 s=CLOSED pgs=183 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:35:16.897 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:16.895+0000 7f18fa5a5700 1 --2- 172.21.15.66:0/3354041880 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f18f40f5410 0x7f18f40fd3a0 unknown :-1 s=CLOSED pgs=436 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:35:16.897 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:16.895+0000 7f18fa5a5700 1 -- 172.21.15.66:0/3354041880 >> 172.21.15.66:0/3354041880 conn(0x7f18f40f0b70 msgr2=0x7f18f405fe00 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:35:16.897 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:16.895+0000 7f18fa5a5700 1 -- 172.21.15.66:0/3354041880 shutdown_connections 2023-11-17T05:35:16.897 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:16.895+0000 7f18fa5a5700 1 -- 172.21.15.66:0/3354041880 wait complete. 2023-11-17T05:35:16.898 INFO:teuthology.orchestra.run.smithi066.stderr:dumped monmap epoch 1 2023-11-17T05:35:17.239 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:35:16 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:35:16.895176+0000 mon.smithi040 (mon.0) 336 : audit [DBG] from='client.? 172.21.15.66:0/3354041880' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-11-17T05:35:18.239 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:35:17 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:35:17.554563+0000 mgr.smithi040.ogcprl (mgr.14164) 434 : cluster [DBG] pgmap v418: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:35:18.314 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2023-11-17T05:35:18.314 DEBUG:teuthology.orchestra.run.smithi066:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.4 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ed6470e2-8508-11ee-8dbe-212e2dc638e7 -- ceph mon dump -f json 2023-11-17T05:35:19.539 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:19.530+0000 7f9521b3a700 1 -- 172.21.15.66:0/2398846341 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f951c060280 msgr2=0x7f951c0f90f0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:35:19.539 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:19.530+0000 7f9521b3a700 1 --2- 172.21.15.66:0/2398846341 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f951c060280 0x7f951c0f90f0 secure :-1 s=READY pgs=437 cs=0 l=1 rev1=1 rx=0x7f950c009fb0 tx=0x7f950c009b30).stop 2023-11-17T05:35:19.539 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:19.531+0000 7f9521b3a700 1 -- 172.21.15.66:0/2398846341 shutdown_connections 2023-11-17T05:35:19.539 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:19.531+0000 7f9521b3a700 1 --2- 172.21.15.66:0/2398846341 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f951c060280 0x7f951c0f90f0 unknown :-1 s=CLOSED pgs=437 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:35:19.540 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:19.531+0000 7f9521b3a700 1 -- 172.21.15.66:0/2398846341 >> 172.21.15.66:0/2398846341 conn(0x7f951c0f0b70 msgr2=0x7f951c0f2f90 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:35:19.540 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:19.531+0000 7f9521b3a700 1 -- 172.21.15.66:0/2398846341 shutdown_connections 2023-11-17T05:35:19.540 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:19.531+0000 7f9521b3a700 1 -- 172.21.15.66:0/2398846341 wait complete. 2023-11-17T05:35:19.540 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:19.531+0000 7f9521b3a700 1 Processor -- start 2023-11-17T05:35:19.540 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:19.531+0000 7f9521b3a700 1 -- start start 2023-11-17T05:35:19.540 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:19.532+0000 7f9521b3a700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f951c060280 0x7f951c1066e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:35:19.540 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:19.532+0000 7f9521b3a700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f951c106bd0 con 0x7f951c060280 2023-11-17T05:35:19.540 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:19.532+0000 7f951b7fe700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f951c060280 0x7f951c1066e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:35:19.540 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:19.532+0000 7f951b7fe700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f951c060280 0x7f951c1066e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.66:54662/0 (socket says 172.21.15.66:54662) 2023-11-17T05:35:19.540 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:19.532+0000 7f951b7fe700 1 -- 172.21.15.66:0/3743467888 learned_addr learned my addr 172.21.15.66:0/3743467888 (peer_addr_for_me v2:172.21.15.66:0/0) 2023-11-17T05:35:19.540 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:19.532+0000 7f951b7fe700 1 -- 172.21.15.66:0/3743467888 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f950c005040 con 0x7f951c060280 2023-11-17T05:35:19.541 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:19.532+0000 7f951b7fe700 1 --2- 172.21.15.66:0/3743467888 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f951c060280 0x7f951c1066e0 secure :-1 s=READY pgs=438 cs=0 l=1 rev1=1 rx=0x7f950c013510 tx=0x7f950c00b040).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:35:19.541 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:19.532+0000 7f9503fff700 1 -- 172.21.15.66:0/3743467888 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f950c009320 con 0x7f951c060280 2023-11-17T05:35:19.541 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:19.533+0000 7f9503fff700 1 -- 172.21.15.66:0/3743467888 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(25 keys) v1 ==== 1074+0+0 (secure 0 0 0) 0x7f950c013f00 con 0x7f951c060280 2023-11-17T05:35:19.541 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:19.533+0000 7f9503fff700 1 -- 172.21.15.66:0/3743467888 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f950c014dc0 con 0x7f951c060280 2023-11-17T05:35:19.541 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:19.533+0000 7f9521b3a700 1 -- 172.21.15.66:0/3743467888 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f951c106dd0 con 0x7f951c060280 2023-11-17T05:35:19.541 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:19.533+0000 7f9521b3a700 1 -- 172.21.15.66:0/3743467888 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f951c107260 con 0x7f951c060280 2023-11-17T05:35:19.541 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:19.534+0000 7f9503fff700 1 -- 172.21.15.66:0/3743467888 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 17) v1 ==== 36269+0+0 (secure 0 0 0) 0x7f950c0095f0 con 0x7f951c060280 2023-11-17T05:35:19.541 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:19.534+0000 7f9521b3a700 1 -- 172.21.15.66:0/3743467888 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f951c045ad0 con 0x7f951c060280 2023-11-17T05:35:19.541 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:19.534+0000 7f9503fff700 1 --2- 172.21.15.66:0/3743467888 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f950402ff60 0x7f9504032420 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:35:19.541 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:19.534+0000 7f9503fff700 1 -- 172.21.15.66:0/3743467888 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(5..5 src has 1..5) v4 ==== 1270+0+0 (secure 0 0 0) 0x7f950c041040 con 0x7f951c060280 2023-11-17T05:35:19.541 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:19.534+0000 7f951affd700 1 --2- 172.21.15.66:0/3743467888 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f950402ff60 0x7f9504032420 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:35:19.541 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:19.534+0000 7f951affd700 1 --2- 172.21.15.66:0/3743467888 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f950402ff60 0x7f9504032420 secure :-1 s=READY pgs=184 cs=0 l=1 rev1=1 rx=0x7f95100096a0 tx=0x7f9510006b40).ready entity=mgr.14164 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:35:19.542 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:19.539+0000 7f9503fff700 1 -- 172.21.15.66:0/3743467888 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+152212 (secure 0 0 0) 0x7f950c014420 con 0x7f951c060280 2023-11-17T05:35:19.758 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:19.757+0000 7f9521b3a700 1 -- 172.21.15.66:0/3743467888 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "mon dump", "format": "json"} v 0) v1 -- 0x7f951c108110 con 0x7f951c060280 2023-11-17T05:35:19.758 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:19.757+0000 7f9503fff700 1 -- 172.21.15.66:0/3743467888 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mon_command_ack([{"prefix": "mon dump", "format": "json"}]=0 dumped monmap epoch 1 v1) v1 ==== 95+0+690 (secure 0 0 0) 0x7f950c010230 con 0x7f951c060280 2023-11-17T05:35:19.759 INFO:teuthology.orchestra.run.smithi066.stdout: 2023-11-17T05:35:19.759 INFO:teuthology.orchestra.run.smithi066.stdout:{"epoch":1,"fsid":"ed6470e2-8508-11ee-8dbe-212e2dc638e7","modified":"2023-11-17T05:20:13.934038Z","created":"2023-11-17T05:20:13.934038Z","min_mon_release":16,"min_mon_release_name":"pacific","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging"],"optional":[]},"mons":[{"rank":0,"name":"smithi040","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:3300","nonce":0},{"type":"v1","addr":"172.21.15.40:6789","nonce":0}]},"addr":"172.21.15.40:6789/0","public_addr":"172.21.15.40:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2023-11-17T05:35:19.760 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:19.759+0000 7f9521b3a700 1 -- 172.21.15.66:0/3743467888 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f950402ff60 msgr2=0x7f9504032420 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:35:19.760 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:19.759+0000 7f9521b3a700 1 --2- 172.21.15.66:0/3743467888 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f950402ff60 0x7f9504032420 secure :-1 s=READY pgs=184 cs=0 l=1 rev1=1 rx=0x7f95100096a0 tx=0x7f9510006b40).stop 2023-11-17T05:35:19.761 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:19.759+0000 7f9521b3a700 1 -- 172.21.15.66:0/3743467888 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f951c060280 msgr2=0x7f951c1066e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:35:19.761 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:19.759+0000 7f9521b3a700 1 --2- 172.21.15.66:0/3743467888 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f951c060280 0x7f951c1066e0 secure :-1 s=READY pgs=438 cs=0 l=1 rev1=1 rx=0x7f950c013510 tx=0x7f950c00b040).stop 2023-11-17T05:35:19.761 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:19.759+0000 7f9521b3a700 1 -- 172.21.15.66:0/3743467888 shutdown_connections 2023-11-17T05:35:19.761 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:19.759+0000 7f9521b3a700 1 --2- 172.21.15.66:0/3743467888 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f950402ff60 0x7f9504032420 unknown :-1 s=CLOSED pgs=184 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:35:19.761 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:19.759+0000 7f9521b3a700 1 --2- 172.21.15.66:0/3743467888 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f951c060280 0x7f951c1066e0 unknown :-1 s=CLOSED pgs=438 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:35:19.761 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:19.759+0000 7f9521b3a700 1 -- 172.21.15.66:0/3743467888 >> 172.21.15.66:0/3743467888 conn(0x7f951c0f0b70 msgr2=0x7f951c0f1660 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:35:19.761 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:19.759+0000 7f9521b3a700 1 -- 172.21.15.66:0/3743467888 shutdown_connections 2023-11-17T05:35:19.761 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:19.759+0000 7f9521b3a700 1 -- 172.21.15.66:0/3743467888 wait complete. 2023-11-17T05:35:19.761 INFO:teuthology.orchestra.run.smithi066.stderr:dumped monmap epoch 1 2023-11-17T05:35:20.989 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:35:20 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:35:19.554927+0000 mgr.smithi040.ogcprl (mgr.14164) 435 : cluster [DBG] pgmap v419: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:35:20.989 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:35:20 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:35:19.759143+0000 mon.smithi040 (mon.0) 337 : audit [DBG] from='client.? 172.21.15.66:0/3743467888' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-11-17T05:35:21.157 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2023-11-17T05:35:21.157 DEBUG:teuthology.orchestra.run.smithi066:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.4 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ed6470e2-8508-11ee-8dbe-212e2dc638e7 -- ceph mon dump -f json 2023-11-17T05:35:21.989 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:35:21 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:35:21.555168+0000 mgr.smithi040.ogcprl (mgr.14164) 436 : cluster [DBG] pgmap v420: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:35:22.356 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:22.353+0000 7fe3b7097700 1 -- 172.21.15.66:0/1801613619 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fe3b00f47c0 msgr2=0x7fe3b00f4be0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:35:22.357 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:22.353+0000 7fe3b7097700 1 --2- 172.21.15.66:0/1801613619 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fe3b00f47c0 0x7fe3b00f4be0 secure :-1 s=READY pgs=439 cs=0 l=1 rev1=1 rx=0x7fe3a4009fb0 tx=0x7fe3a4009b30).stop 2023-11-17T05:35:22.357 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:22.353+0000 7fe3b7097700 1 -- 172.21.15.66:0/1801613619 shutdown_connections 2023-11-17T05:35:22.357 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:22.353+0000 7fe3b7097700 1 --2- 172.21.15.66:0/1801613619 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fe3b00f47c0 0x7fe3b00f4be0 unknown :-1 s=CLOSED pgs=439 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:35:22.357 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:22.353+0000 7fe3b7097700 1 -- 172.21.15.66:0/1801613619 >> 172.21.15.66:0/1801613619 conn(0x7fe3b00f02e0 msgr2=0x7fe3b00f2700 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:35:22.357 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:22.353+0000 7fe3b7097700 1 -- 172.21.15.66:0/1801613619 shutdown_connections 2023-11-17T05:35:22.357 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:22.353+0000 7fe3b7097700 1 -- 172.21.15.66:0/1801613619 wait complete. 2023-11-17T05:35:22.357 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:22.354+0000 7fe3b7097700 1 Processor -- start 2023-11-17T05:35:22.357 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:22.354+0000 7fe3b7097700 1 -- start start 2023-11-17T05:35:22.357 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:22.354+0000 7fe3b7097700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fe3b00f47c0 0x7fe3b00f7370 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:35:22.357 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:22.354+0000 7fe3b7097700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fe3b00f57f0 con 0x7fe3b00f47c0 2023-11-17T05:35:22.357 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:22.354+0000 7fe3b4e33700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fe3b00f47c0 0x7fe3b00f7370 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:35:22.357 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:22.354+0000 7fe3b4e33700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fe3b00f47c0 0x7fe3b00f7370 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.66:33390/0 (socket says 172.21.15.66:33390) 2023-11-17T05:35:22.358 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:22.354+0000 7fe3b4e33700 1 -- 172.21.15.66:0/3696624678 learned_addr learned my addr 172.21.15.66:0/3696624678 (peer_addr_for_me v2:172.21.15.66:0/0) 2023-11-17T05:35:22.358 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:22.355+0000 7fe3b4e33700 1 -- 172.21.15.66:0/3696624678 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fe3a4005040 con 0x7fe3b00f47c0 2023-11-17T05:35:22.358 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:22.355+0000 7fe3b4e33700 1 --2- 172.21.15.66:0/3696624678 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fe3b00f47c0 0x7fe3b00f7370 secure :-1 s=READY pgs=440 cs=0 l=1 rev1=1 rx=0x7fe3a4013510 tx=0x7fe3a400b040).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:35:22.358 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:22.355+0000 7fe3ad7fa700 1 -- 172.21.15.66:0/3696624678 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7fe3a4009320 con 0x7fe3b00f47c0 2023-11-17T05:35:22.358 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:22.355+0000 7fe3b7097700 1 -- 172.21.15.66:0/3696624678 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fe3b00f5990 con 0x7fe3b00f47c0 2023-11-17T05:35:22.358 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:22.355+0000 7fe3ad7fa700 1 -- 172.21.15.66:0/3696624678 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(25 keys) v1 ==== 1074+0+0 (secure 0 0 0) 0x7fe3a4013f00 con 0x7fe3b00f47c0 2023-11-17T05:35:22.358 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:22.355+0000 7fe3ad7fa700 1 -- 172.21.15.66:0/3696624678 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7fe3a4014d90 con 0x7fe3b00f47c0 2023-11-17T05:35:22.358 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:22.355+0000 7fe3b7097700 1 -- 172.21.15.66:0/3696624678 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fe3b00f5e20 con 0x7fe3b00f47c0 2023-11-17T05:35:22.359 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:22.356+0000 7fe3ad7fa700 1 -- 172.21.15.66:0/3696624678 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 17) v1 ==== 36269+0+0 (secure 0 0 0) 0x7fe3a4014420 con 0x7fe3b00f47c0 2023-11-17T05:35:22.359 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:22.356+0000 7fe3b7097700 1 -- 172.21.15.66:0/3696624678 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fe3b0059100 con 0x7fe3b00f47c0 2023-11-17T05:35:22.359 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:22.356+0000 7fe3ad7fa700 1 --2- 172.21.15.66:0/3696624678 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fe3a002ff60 0x7fe3a0032420 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:35:22.359 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:22.356+0000 7fe3ad7fa700 1 -- 172.21.15.66:0/3696624678 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(5..5 src has 1..5) v4 ==== 1270+0+0 (secure 0 0 0) 0x7fe3a4041110 con 0x7fe3b00f47c0 2023-11-17T05:35:22.359 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:22.356+0000 7fe3affff700 1 --2- 172.21.15.66:0/3696624678 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fe3a002ff60 0x7fe3a0032420 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:35:22.359 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:22.357+0000 7fe3affff700 1 --2- 172.21.15.66:0/3696624678 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fe3a002ff60 0x7fe3a0032420 secure :-1 s=READY pgs=185 cs=0 l=1 rev1=1 rx=0x7fe39c0096a0 tx=0x7fe39c006b40).ready entity=mgr.14164 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:35:22.366 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:22.364+0000 7fe3ad7fa700 1 -- 172.21.15.66:0/3696624678 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+152212 (secure 0 0 0) 0x7fe3a40095f0 con 0x7fe3b00f47c0 2023-11-17T05:35:22.585 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:22.583+0000 7fe3b7097700 1 -- 172.21.15.66:0/3696624678 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "mon dump", "format": "json"} v 0) v1 -- 0x7fe3b00fbf60 con 0x7fe3b00f47c0 2023-11-17T05:35:22.585 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:22.584+0000 7fe3ad7fa700 1 -- 172.21.15.66:0/3696624678 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mon_command_ack([{"prefix": "mon dump", "format": "json"}]=0 dumped monmap epoch 1 v1) v1 ==== 95+0+690 (secure 0 0 0) 0x7fe3a401db70 con 0x7fe3b00f47c0 2023-11-17T05:35:22.586 INFO:teuthology.orchestra.run.smithi066.stdout: 2023-11-17T05:35:22.586 INFO:teuthology.orchestra.run.smithi066.stdout:{"epoch":1,"fsid":"ed6470e2-8508-11ee-8dbe-212e2dc638e7","modified":"2023-11-17T05:20:13.934038Z","created":"2023-11-17T05:20:13.934038Z","min_mon_release":16,"min_mon_release_name":"pacific","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging"],"optional":[]},"mons":[{"rank":0,"name":"smithi040","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:3300","nonce":0},{"type":"v1","addr":"172.21.15.40:6789","nonce":0}]},"addr":"172.21.15.40:6789/0","public_addr":"172.21.15.40:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2023-11-17T05:35:22.587 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:22.586+0000 7fe3b7097700 1 -- 172.21.15.66:0/3696624678 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fe3a002ff60 msgr2=0x7fe3a0032420 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:35:22.587 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:22.586+0000 7fe3b7097700 1 --2- 172.21.15.66:0/3696624678 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fe3a002ff60 0x7fe3a0032420 secure :-1 s=READY pgs=185 cs=0 l=1 rev1=1 rx=0x7fe39c0096a0 tx=0x7fe39c006b40).stop 2023-11-17T05:35:22.587 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:22.586+0000 7fe3b7097700 1 -- 172.21.15.66:0/3696624678 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fe3b00f47c0 msgr2=0x7fe3b00f7370 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:35:22.587 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:22.586+0000 7fe3b7097700 1 --2- 172.21.15.66:0/3696624678 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fe3b00f47c0 0x7fe3b00f7370 secure :-1 s=READY pgs=440 cs=0 l=1 rev1=1 rx=0x7fe3a4013510 tx=0x7fe3a400b040).stop 2023-11-17T05:35:22.588 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:22.586+0000 7fe3b7097700 1 -- 172.21.15.66:0/3696624678 shutdown_connections 2023-11-17T05:35:22.588 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:22.586+0000 7fe3b7097700 1 --2- 172.21.15.66:0/3696624678 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fe3a002ff60 0x7fe3a0032420 unknown :-1 s=CLOSED pgs=185 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:35:22.588 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:22.586+0000 7fe3b7097700 1 --2- 172.21.15.66:0/3696624678 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fe3b00f47c0 0x7fe3b00f7370 unknown :-1 s=CLOSED pgs=440 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:35:22.588 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:22.586+0000 7fe3b7097700 1 -- 172.21.15.66:0/3696624678 >> 172.21.15.66:0/3696624678 conn(0x7fe3b00f02e0 msgr2=0x7fe3b00f0db0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:35:22.588 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:22.586+0000 7fe3b7097700 1 -- 172.21.15.66:0/3696624678 shutdown_connections 2023-11-17T05:35:22.588 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:22.586+0000 7fe3b7097700 1 -- 172.21.15.66:0/3696624678 wait complete. 2023-11-17T05:35:22.588 INFO:teuthology.orchestra.run.smithi066.stderr:dumped monmap epoch 1 2023-11-17T05:35:22.988 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:35:22 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:35:22.585891+0000 mon.smithi040 (mon.0) 338 : audit [DBG] from='client.? 172.21.15.66:0/3696624678' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-11-17T05:35:23.960 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2023-11-17T05:35:23.960 DEBUG:teuthology.orchestra.run.smithi066:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.4 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ed6470e2-8508-11ee-8dbe-212e2dc638e7 -- ceph mon dump -f json 2023-11-17T05:35:23.988 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:35:23 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:35:23.555511+0000 mgr.smithi040.ogcprl (mgr.14164) 437 : cluster [DBG] pgmap v421: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:35:25.167 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:25.165+0000 7f5f3d4e1700 1 -- 172.21.15.66:0/3236576053 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f5f38060220 msgr2=0x7f5f380f9150 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:35:25.167 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:25.165+0000 7f5f3d4e1700 1 --2- 172.21.15.66:0/3236576053 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f5f38060220 0x7f5f380f9150 secure :-1 s=READY pgs=441 cs=0 l=1 rev1=1 rx=0x7f5f2c009fb0 tx=0x7f5f2c009b30).stop 2023-11-17T05:35:25.168 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:25.165+0000 7f5f3d4e1700 1 -- 172.21.15.66:0/3236576053 shutdown_connections 2023-11-17T05:35:25.168 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:25.165+0000 7f5f3d4e1700 1 --2- 172.21.15.66:0/3236576053 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f5f38060220 0x7f5f380f9150 unknown :-1 s=CLOSED pgs=441 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:35:25.168 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:25.165+0000 7f5f3d4e1700 1 -- 172.21.15.66:0/3236576053 >> 172.21.15.66:0/3236576053 conn(0x7f5f380f0b70 msgr2=0x7f5f380f2f90 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:35:25.168 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:25.165+0000 7f5f3d4e1700 1 -- 172.21.15.66:0/3236576053 shutdown_connections 2023-11-17T05:35:25.168 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:25.165+0000 7f5f3d4e1700 1 -- 172.21.15.66:0/3236576053 wait complete. 2023-11-17T05:35:25.168 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:25.166+0000 7f5f3d4e1700 1 Processor -- start 2023-11-17T05:35:25.168 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:25.166+0000 7f5f3d4e1700 1 -- start start 2023-11-17T05:35:25.168 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:25.166+0000 7f5f3d4e1700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f5f38060220 0x7f5f380f6bd0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:35:25.168 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:25.166+0000 7f5f3d4e1700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f5f380f5050 con 0x7f5f38060220 2023-11-17T05:35:25.168 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:25.166+0000 7f5f36ffd700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f5f38060220 0x7f5f380f6bd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:35:25.168 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:25.166+0000 7f5f36ffd700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f5f38060220 0x7f5f380f6bd0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.66:33412/0 (socket says 172.21.15.66:33412) 2023-11-17T05:35:25.168 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:25.166+0000 7f5f36ffd700 1 -- 172.21.15.66:0/2820446294 learned_addr learned my addr 172.21.15.66:0/2820446294 (peer_addr_for_me v2:172.21.15.66:0/0) 2023-11-17T05:35:25.170 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:25.167+0000 7f5f36ffd700 1 -- 172.21.15.66:0/2820446294 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f5f2c005040 con 0x7f5f38060220 2023-11-17T05:35:25.170 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:25.167+0000 7f5f36ffd700 1 --2- 172.21.15.66:0/2820446294 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f5f38060220 0x7f5f380f6bd0 secure :-1 s=READY pgs=442 cs=0 l=1 rev1=1 rx=0x7f5f2c013510 tx=0x7f5f2c00b040).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:35:25.170 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:25.167+0000 7f5f1f7fe700 1 -- 172.21.15.66:0/2820446294 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f5f2c009320 con 0x7f5f38060220 2023-11-17T05:35:25.170 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:25.167+0000 7f5f1f7fe700 1 -- 172.21.15.66:0/2820446294 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(25 keys) v1 ==== 1074+0+0 (secure 0 0 0) 0x7f5f2c013f00 con 0x7f5f38060220 2023-11-17T05:35:25.170 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:25.167+0000 7f5f3d4e1700 1 -- 172.21.15.66:0/2820446294 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f5f380f51f0 con 0x7f5f38060220 2023-11-17T05:35:25.170 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:25.167+0000 7f5f1f7fe700 1 -- 172.21.15.66:0/2820446294 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7f5f2c014dc0 con 0x7f5f38060220 2023-11-17T05:35:25.170 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:25.167+0000 7f5f3d4e1700 1 -- 172.21.15.66:0/2820446294 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f5f380f5680 con 0x7f5f38060220 2023-11-17T05:35:25.170 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:25.168+0000 7f5f1f7fe700 1 -- 172.21.15.66:0/2820446294 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 17) v1 ==== 36269+0+0 (secure 0 0 0) 0x7f5f2c0095f0 con 0x7f5f38060220 2023-11-17T05:35:25.171 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:25.168+0000 7f5f3d4e1700 1 -- 172.21.15.66:0/2820446294 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f5f38059100 con 0x7f5f38060220 2023-11-17T05:35:25.171 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:25.168+0000 7f5f1f7fe700 1 --2- 172.21.15.66:0/2820446294 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f5f2002ffb0 0x7f5f20032470 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:35:25.171 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:25.168+0000 7f5f1f7fe700 1 -- 172.21.15.66:0/2820446294 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(5..5 src has 1..5) v4 ==== 1270+0+0 (secure 0 0 0) 0x7f5f2c040ff0 con 0x7f5f38060220 2023-11-17T05:35:25.171 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:25.168+0000 7f5f367fc700 1 --2- 172.21.15.66:0/2820446294 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f5f2002ffb0 0x7f5f20032470 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:35:25.171 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:25.169+0000 7f5f367fc700 1 --2- 172.21.15.66:0/2820446294 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f5f2002ffb0 0x7f5f20032470 secure :-1 s=READY pgs=186 cs=0 l=1 rev1=1 rx=0x7f5f280096a0 tx=0x7f5f28006b40).ready entity=mgr.14164 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:35:25.174 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:25.173+0000 7f5f1f7fe700 1 -- 172.21.15.66:0/2820446294 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+152212 (secure 0 0 0) 0x7f5f2c06d2b0 con 0x7f5f38060220 2023-11-17T05:35:25.393 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:25.392+0000 7f5f3d4e1700 1 -- 172.21.15.66:0/2820446294 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "mon dump", "format": "json"} v 0) v1 -- 0x7f5f380fbf50 con 0x7f5f38060220 2023-11-17T05:35:25.394 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:25.393+0000 7f5f1f7fe700 1 -- 172.21.15.66:0/2820446294 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mon_command_ack([{"prefix": "mon dump", "format": "json"}]=0 dumped monmap epoch 1 v1) v1 ==== 95+0+690 (secure 0 0 0) 0x7f5f2c014420 con 0x7f5f38060220 2023-11-17T05:35:25.394 INFO:teuthology.orchestra.run.smithi066.stdout: 2023-11-17T05:35:25.394 INFO:teuthology.orchestra.run.smithi066.stdout:{"epoch":1,"fsid":"ed6470e2-8508-11ee-8dbe-212e2dc638e7","modified":"2023-11-17T05:20:13.934038Z","created":"2023-11-17T05:20:13.934038Z","min_mon_release":16,"min_mon_release_name":"pacific","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging"],"optional":[]},"mons":[{"rank":0,"name":"smithi040","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:3300","nonce":0},{"type":"v1","addr":"172.21.15.40:6789","nonce":0}]},"addr":"172.21.15.40:6789/0","public_addr":"172.21.15.40:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2023-11-17T05:35:25.396 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:25.394+0000 7f5f3d4e1700 1 -- 172.21.15.66:0/2820446294 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f5f2002ffb0 msgr2=0x7f5f20032470 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:35:25.396 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:25.394+0000 7f5f3d4e1700 1 --2- 172.21.15.66:0/2820446294 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f5f2002ffb0 0x7f5f20032470 secure :-1 s=READY pgs=186 cs=0 l=1 rev1=1 rx=0x7f5f280096a0 tx=0x7f5f28006b40).stop 2023-11-17T05:35:25.396 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:25.394+0000 7f5f3d4e1700 1 -- 172.21.15.66:0/2820446294 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f5f38060220 msgr2=0x7f5f380f6bd0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:35:25.396 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:25.394+0000 7f5f3d4e1700 1 --2- 172.21.15.66:0/2820446294 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f5f38060220 0x7f5f380f6bd0 secure :-1 s=READY pgs=442 cs=0 l=1 rev1=1 rx=0x7f5f2c013510 tx=0x7f5f2c00b040).stop 2023-11-17T05:35:25.396 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:25.394+0000 7f5f3d4e1700 1 -- 172.21.15.66:0/2820446294 shutdown_connections 2023-11-17T05:35:25.396 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:25.394+0000 7f5f3d4e1700 1 --2- 172.21.15.66:0/2820446294 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f5f2002ffb0 0x7f5f20032470 unknown :-1 s=CLOSED pgs=186 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:35:25.396 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:25.394+0000 7f5f3d4e1700 1 --2- 172.21.15.66:0/2820446294 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f5f38060220 0x7f5f380f6bd0 unknown :-1 s=CLOSED pgs=442 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:35:25.396 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:25.394+0000 7f5f3d4e1700 1 -- 172.21.15.66:0/2820446294 >> 172.21.15.66:0/2820446294 conn(0x7f5f380f0b70 msgr2=0x7f5f380f1650 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:35:25.396 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:25.394+0000 7f5f3d4e1700 1 -- 172.21.15.66:0/2820446294 shutdown_connections 2023-11-17T05:35:25.396 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:25.395+0000 7f5f3d4e1700 1 -- 172.21.15.66:0/2820446294 wait complete. 2023-11-17T05:35:25.397 INFO:teuthology.orchestra.run.smithi066.stderr:dumped monmap epoch 1 2023-11-17T05:35:25.989 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:35:25 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:35:25.394367+0000 mon.smithi040 (mon.0) 339 : audit [DBG] from='client.? 172.21.15.66:0/2820446294' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-11-17T05:35:26.780 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2023-11-17T05:35:26.781 DEBUG:teuthology.orchestra.run.smithi066:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.4 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ed6470e2-8508-11ee-8dbe-212e2dc638e7 -- ceph mon dump -f json 2023-11-17T05:35:26.988 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:35:26 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:35:25.555896+0000 mgr.smithi040.ogcprl (mgr.14164) 438 : cluster [DBG] pgmap v422: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:35:27.991 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:35:27 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:35:27.556299+0000 mgr.smithi040.ogcprl (mgr.14164) 439 : cluster [DBG] pgmap v423: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:35:28.000 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:27.998+0000 7fe01f16e700 1 -- 172.21.15.66:0/3886075699 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fe0180f78e0 msgr2=0x7fe0180f7d00 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:35:28.001 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:27.998+0000 7fe01f16e700 1 --2- 172.21.15.66:0/3886075699 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fe0180f78e0 0x7fe0180f7d00 secure :-1 s=READY pgs=443 cs=0 l=1 rev1=1 rx=0x7fe014009fb0 tx=0x7fe014009b30).stop 2023-11-17T05:35:28.001 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:27.999+0000 7fe01f16e700 1 -- 172.21.15.66:0/3886075699 shutdown_connections 2023-11-17T05:35:28.001 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:27.999+0000 7fe01f16e700 1 --2- 172.21.15.66:0/3886075699 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fe0180f78e0 0x7fe0180f7d00 unknown :-1 s=CLOSED pgs=443 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:35:28.001 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:27.999+0000 7fe01f16e700 1 -- 172.21.15.66:0/3886075699 >> 172.21.15.66:0/3886075699 conn(0x7fe0180f0120 msgr2=0x7fe0180f2540 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:35:28.001 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:27.999+0000 7fe01f16e700 1 -- 172.21.15.66:0/3886075699 shutdown_connections 2023-11-17T05:35:28.001 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:27.999+0000 7fe01f16e700 1 -- 172.21.15.66:0/3886075699 wait complete. 2023-11-17T05:35:28.001 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:27.999+0000 7fe01f16e700 1 Processor -- start 2023-11-17T05:35:28.001 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:27.999+0000 7fe01f16e700 1 -- start start 2023-11-17T05:35:28.001 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:27.999+0000 7fe01f16e700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fe0180f78e0 0x7fe0181064e0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:35:28.001 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:27.999+0000 7fe01f16e700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fe0181069d0 con 0x7fe0180f78e0 2023-11-17T05:35:28.003 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:28.000+0000 7fe01e16c700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fe0180f78e0 0x7fe0181064e0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:35:28.003 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:28.000+0000 7fe01e16c700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fe0180f78e0 0x7fe0181064e0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.66:33430/0 (socket says 172.21.15.66:33430) 2023-11-17T05:35:28.003 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:28.000+0000 7fe01e16c700 1 -- 172.21.15.66:0/3305591292 learned_addr learned my addr 172.21.15.66:0/3305591292 (peer_addr_for_me v2:172.21.15.66:0/0) 2023-11-17T05:35:28.003 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:28.000+0000 7fe01e16c700 1 -- 172.21.15.66:0/3305591292 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fe014005040 con 0x7fe0180f78e0 2023-11-17T05:35:28.003 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:28.000+0000 7fe01e16c700 1 --2- 172.21.15.66:0/3305591292 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fe0180f78e0 0x7fe0181064e0 secure :-1 s=READY pgs=444 cs=0 l=1 rev1=1 rx=0x7fe014013510 tx=0x7fe01400b040).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:35:28.003 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:28.000+0000 7fe00effd700 1 -- 172.21.15.66:0/3305591292 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7fe014009320 con 0x7fe0180f78e0 2023-11-17T05:35:28.003 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:28.000+0000 7fe01f16e700 1 -- 172.21.15.66:0/3305591292 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fe018106bd0 con 0x7fe0180f78e0 2023-11-17T05:35:28.003 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:28.000+0000 7fe00effd700 1 -- 172.21.15.66:0/3305591292 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(25 keys) v1 ==== 1074+0+0 (secure 0 0 0) 0x7fe014013f00 con 0x7fe0180f78e0 2023-11-17T05:35:28.003 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:28.000+0000 7fe00effd700 1 -- 172.21.15.66:0/3305591292 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7fe014014dc0 con 0x7fe0180f78e0 2023-11-17T05:35:28.004 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:28.000+0000 7fe01f16e700 1 -- 172.21.15.66:0/3305591292 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fe018107060 con 0x7fe0180f78e0 2023-11-17T05:35:28.004 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:28.001+0000 7fe00effd700 1 -- 172.21.15.66:0/3305591292 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 17) v1 ==== 36269+0+0 (secure 0 0 0) 0x7fe0140095f0 con 0x7fe0180f78e0 2023-11-17T05:35:28.004 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:28.001+0000 7fe01f16e700 1 -- 172.21.15.66:0/3305591292 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fe018045ad0 con 0x7fe0180f78e0 2023-11-17T05:35:28.004 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:28.001+0000 7fe00effd700 1 --2- 172.21.15.66:0/3305591292 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fe00402ff60 0x7fe004032420 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:35:28.004 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:28.001+0000 7fe00effd700 1 -- 172.21.15.66:0/3305591292 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(5..5 src has 1..5) v4 ==== 1270+0+0 (secure 0 0 0) 0x7fe014041080 con 0x7fe0180f78e0 2023-11-17T05:35:28.004 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:28.002+0000 7fe01d96b700 1 --2- 172.21.15.66:0/3305591292 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fe00402ff60 0x7fe004032420 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:35:28.004 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:28.002+0000 7fe01d96b700 1 --2- 172.21.15.66:0/3305591292 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fe00402ff60 0x7fe004032420 secure :-1 s=READY pgs=187 cs=0 l=1 rev1=1 rx=0x7fe0080096a0 tx=0x7fe008006b40).ready entity=mgr.14164 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:35:28.007 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:28.006+0000 7fe00effd700 1 -- 172.21.15.66:0/3305591292 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+152212 (secure 0 0 0) 0x7fe014014420 con 0x7fe0180f78e0 2023-11-17T05:35:28.230 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:28.228+0000 7fe01f16e700 1 -- 172.21.15.66:0/3305591292 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "mon dump", "format": "json"} v 0) v1 -- 0x7fe018107f10 con 0x7fe0180f78e0 2023-11-17T05:35:28.230 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:28.229+0000 7fe00effd700 1 -- 172.21.15.66:0/3305591292 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mon_command_ack([{"prefix": "mon dump", "format": "json"}]=0 dumped monmap epoch 1 v1) v1 ==== 95+0+690 (secure 0 0 0) 0x7fe014010230 con 0x7fe0180f78e0 2023-11-17T05:35:28.231 INFO:teuthology.orchestra.run.smithi066.stdout: 2023-11-17T05:35:28.231 INFO:teuthology.orchestra.run.smithi066.stdout:{"epoch":1,"fsid":"ed6470e2-8508-11ee-8dbe-212e2dc638e7","modified":"2023-11-17T05:20:13.934038Z","created":"2023-11-17T05:20:13.934038Z","min_mon_release":16,"min_mon_release_name":"pacific","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging"],"optional":[]},"mons":[{"rank":0,"name":"smithi040","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:3300","nonce":0},{"type":"v1","addr":"172.21.15.40:6789","nonce":0}]},"addr":"172.21.15.40:6789/0","public_addr":"172.21.15.40:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2023-11-17T05:35:28.232 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:28.231+0000 7fe01f16e700 1 -- 172.21.15.66:0/3305591292 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fe00402ff60 msgr2=0x7fe004032420 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:35:28.232 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:28.231+0000 7fe01f16e700 1 --2- 172.21.15.66:0/3305591292 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fe00402ff60 0x7fe004032420 secure :-1 s=READY pgs=187 cs=0 l=1 rev1=1 rx=0x7fe0080096a0 tx=0x7fe008006b40).stop 2023-11-17T05:35:28.233 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:28.231+0000 7fe01f16e700 1 -- 172.21.15.66:0/3305591292 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fe0180f78e0 msgr2=0x7fe0181064e0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:35:28.233 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:28.231+0000 7fe01f16e700 1 --2- 172.21.15.66:0/3305591292 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fe0180f78e0 0x7fe0181064e0 secure :-1 s=READY pgs=444 cs=0 l=1 rev1=1 rx=0x7fe014013510 tx=0x7fe01400b040).stop 2023-11-17T05:35:28.233 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:28.231+0000 7fe01f16e700 1 -- 172.21.15.66:0/3305591292 shutdown_connections 2023-11-17T05:35:28.233 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:28.231+0000 7fe01f16e700 1 --2- 172.21.15.66:0/3305591292 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fe00402ff60 0x7fe004032420 unknown :-1 s=CLOSED pgs=187 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:35:28.233 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:28.231+0000 7fe01f16e700 1 --2- 172.21.15.66:0/3305591292 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fe0180f78e0 0x7fe0181064e0 unknown :-1 s=CLOSED pgs=444 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:35:28.233 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:28.231+0000 7fe01f16e700 1 -- 172.21.15.66:0/3305591292 >> 172.21.15.66:0/3305591292 conn(0x7fe0180f0120 msgr2=0x7fe0180f0bc0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:35:28.233 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:28.231+0000 7fe01f16e700 1 -- 172.21.15.66:0/3305591292 shutdown_connections 2023-11-17T05:35:28.233 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:28.231+0000 7fe01f16e700 1 -- 172.21.15.66:0/3305591292 wait complete. 2023-11-17T05:35:28.233 INFO:teuthology.orchestra.run.smithi066.stderr:dumped monmap epoch 1 2023-11-17T05:35:28.988 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:35:28 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:35:28.230830+0000 mon.smithi040 (mon.0) 340 : audit [DBG] from='client.? 172.21.15.66:0/3305591292' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-11-17T05:35:29.634 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2023-11-17T05:35:29.634 DEBUG:teuthology.orchestra.run.smithi066:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.4 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ed6470e2-8508-11ee-8dbe-212e2dc638e7 -- ceph mon dump -f json 2023-11-17T05:35:30.827 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:30.823+0000 7fb47ed1f700 1 -- 172.21.15.66:0/190717431 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fb4780f62a0 msgr2=0x7fb4780f66c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:35:30.827 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:30.823+0000 7fb47ed1f700 1 --2- 172.21.15.66:0/190717431 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fb4780f62a0 0x7fb4780f66c0 secure :-1 s=READY pgs=445 cs=0 l=1 rev1=1 rx=0x7fb474009fb0 tx=0x7fb474009b30).stop 2023-11-17T05:35:30.827 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:30.824+0000 7fb47ed1f700 1 -- 172.21.15.66:0/190717431 shutdown_connections 2023-11-17T05:35:30.827 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:30.824+0000 7fb47ed1f700 1 --2- 172.21.15.66:0/190717431 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fb4780f62a0 0x7fb4780f66c0 unknown :-1 s=CLOSED pgs=445 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:35:30.827 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:30.824+0000 7fb47ed1f700 1 -- 172.21.15.66:0/190717431 >> 172.21.15.66:0/190717431 conn(0x7fb4780f1aa0 msgr2=0x7fb4780f3ec0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:35:30.827 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:30.824+0000 7fb47ed1f700 1 -- 172.21.15.66:0/190717431 shutdown_connections 2023-11-17T05:35:30.827 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:30.824+0000 7fb47ed1f700 1 -- 172.21.15.66:0/190717431 wait complete. 2023-11-17T05:35:30.827 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:30.824+0000 7fb47ed1f700 1 Processor -- start 2023-11-17T05:35:30.827 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:30.824+0000 7fb47ed1f700 1 -- start start 2023-11-17T05:35:30.827 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:30.824+0000 7fb47ed1f700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fb4780f62a0 0x7fb4781064b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:35:30.827 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:30.824+0000 7fb47ed1f700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fb4781069a0 con 0x7fb4780f62a0 2023-11-17T05:35:30.828 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:30.825+0000 7fb47dd1d700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fb4780f62a0 0x7fb4781064b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:35:30.828 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:30.825+0000 7fb47dd1d700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fb4780f62a0 0x7fb4781064b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.66:33456/0 (socket says 172.21.15.66:33456) 2023-11-17T05:35:30.828 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:30.825+0000 7fb47dd1d700 1 -- 172.21.15.66:0/2835729631 learned_addr learned my addr 172.21.15.66:0/2835729631 (peer_addr_for_me v2:172.21.15.66:0/0) 2023-11-17T05:35:30.828 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:30.825+0000 7fb47dd1d700 1 -- 172.21.15.66:0/2835729631 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fb474005040 con 0x7fb4780f62a0 2023-11-17T05:35:30.828 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:30.825+0000 7fb47dd1d700 1 --2- 172.21.15.66:0/2835729631 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fb4780f62a0 0x7fb4781064b0 secure :-1 s=READY pgs=446 cs=0 l=1 rev1=1 rx=0x7fb474013510 tx=0x7fb47400b040).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:35:30.828 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:30.825+0000 7fb46a7fc700 1 -- 172.21.15.66:0/2835729631 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7fb474009320 con 0x7fb4780f62a0 2023-11-17T05:35:30.828 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:30.825+0000 7fb47ed1f700 1 -- 172.21.15.66:0/2835729631 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fb478106ba0 con 0x7fb4780f62a0 2023-11-17T05:35:30.828 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:30.825+0000 7fb46a7fc700 1 -- 172.21.15.66:0/2835729631 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(25 keys) v1 ==== 1074+0+0 (secure 0 0 0) 0x7fb474013f00 con 0x7fb4780f62a0 2023-11-17T05:35:30.828 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:30.825+0000 7fb46a7fc700 1 -- 172.21.15.66:0/2835729631 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7fb474014dc0 con 0x7fb4780f62a0 2023-11-17T05:35:30.828 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:30.825+0000 7fb47ed1f700 1 -- 172.21.15.66:0/2835729631 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fb478107030 con 0x7fb4780f62a0 2023-11-17T05:35:30.829 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:30.826+0000 7fb46a7fc700 1 -- 172.21.15.66:0/2835729631 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 17) v1 ==== 36269+0+0 (secure 0 0 0) 0x7fb4740095f0 con 0x7fb4780f62a0 2023-11-17T05:35:30.829 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:30.826+0000 7fb47ed1f700 1 -- 172.21.15.66:0/2835729631 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fb478045ad0 con 0x7fb4780f62a0 2023-11-17T05:35:30.829 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:30.826+0000 7fb46a7fc700 1 --2- 172.21.15.66:0/2835729631 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fb46402ff60 0x7fb464032420 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:35:30.829 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:30.826+0000 7fb46a7fc700 1 -- 172.21.15.66:0/2835729631 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(5..5 src has 1..5) v4 ==== 1270+0+0 (secure 0 0 0) 0x7fb474041100 con 0x7fb4780f62a0 2023-11-17T05:35:30.829 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:30.826+0000 7fb47d51c700 1 --2- 172.21.15.66:0/2835729631 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fb46402ff60 0x7fb464032420 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:35:30.830 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:30.827+0000 7fb47d51c700 1 --2- 172.21.15.66:0/2835729631 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fb46402ff60 0x7fb464032420 secure :-1 s=READY pgs=188 cs=0 l=1 rev1=1 rx=0x7fb46c0096a0 tx=0x7fb46c006b40).ready entity=mgr.14164 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:35:30.833 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:30.831+0000 7fb46a7fc700 1 -- 172.21.15.66:0/2835729631 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+152212 (secure 0 0 0) 0x7fb474014420 con 0x7fb4780f62a0 2023-11-17T05:35:30.988 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:35:30 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:35:29.556698+0000 mgr.smithi040.ogcprl (mgr.14164) 440 : cluster [DBG] pgmap v424: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:35:31.052 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:31.050+0000 7fb47ed1f700 1 -- 172.21.15.66:0/2835729631 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "mon dump", "format": "json"} v 0) v1 -- 0x7fb478107ee0 con 0x7fb4780f62a0 2023-11-17T05:35:31.052 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:31.051+0000 7fb46a7fc700 1 -- 172.21.15.66:0/2835729631 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mon_command_ack([{"prefix": "mon dump", "format": "json"}]=0 dumped monmap epoch 1 v1) v1 ==== 95+0+690 (secure 0 0 0) 0x7fb474010230 con 0x7fb4780f62a0 2023-11-17T05:35:31.053 INFO:teuthology.orchestra.run.smithi066.stdout: 2023-11-17T05:35:31.053 INFO:teuthology.orchestra.run.smithi066.stdout:{"epoch":1,"fsid":"ed6470e2-8508-11ee-8dbe-212e2dc638e7","modified":"2023-11-17T05:20:13.934038Z","created":"2023-11-17T05:20:13.934038Z","min_mon_release":16,"min_mon_release_name":"pacific","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging"],"optional":[]},"mons":[{"rank":0,"name":"smithi040","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:3300","nonce":0},{"type":"v1","addr":"172.21.15.40:6789","nonce":0}]},"addr":"172.21.15.40:6789/0","public_addr":"172.21.15.40:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2023-11-17T05:35:31.055 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:31.053+0000 7fb47ed1f700 1 -- 172.21.15.66:0/2835729631 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fb46402ff60 msgr2=0x7fb464032420 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:35:31.055 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:31.053+0000 7fb47ed1f700 1 --2- 172.21.15.66:0/2835729631 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fb46402ff60 0x7fb464032420 secure :-1 s=READY pgs=188 cs=0 l=1 rev1=1 rx=0x7fb46c0096a0 tx=0x7fb46c006b40).stop 2023-11-17T05:35:31.055 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:31.053+0000 7fb47ed1f700 1 -- 172.21.15.66:0/2835729631 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fb4780f62a0 msgr2=0x7fb4781064b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:35:31.055 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:31.053+0000 7fb47ed1f700 1 --2- 172.21.15.66:0/2835729631 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fb4780f62a0 0x7fb4781064b0 secure :-1 s=READY pgs=446 cs=0 l=1 rev1=1 rx=0x7fb474013510 tx=0x7fb47400b040).stop 2023-11-17T05:35:31.055 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:31.053+0000 7fb47ed1f700 1 -- 172.21.15.66:0/2835729631 shutdown_connections 2023-11-17T05:35:31.055 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:31.053+0000 7fb47ed1f700 1 --2- 172.21.15.66:0/2835729631 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fb46402ff60 0x7fb464032420 unknown :-1 s=CLOSED pgs=188 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:35:31.055 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:31.053+0000 7fb47ed1f700 1 --2- 172.21.15.66:0/2835729631 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fb4780f62a0 0x7fb4781064b0 unknown :-1 s=CLOSED pgs=446 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:35:31.055 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:31.053+0000 7fb47ed1f700 1 -- 172.21.15.66:0/2835729631 >> 172.21.15.66:0/2835729631 conn(0x7fb4780f1aa0 msgr2=0x7fb4780f2510 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:35:31.056 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:31.053+0000 7fb47ed1f700 1 -- 172.21.15.66:0/2835729631 shutdown_connections 2023-11-17T05:35:31.056 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:31.053+0000 7fb47ed1f700 1 -- 172.21.15.66:0/2835729631 wait complete. 2023-11-17T05:35:31.056 INFO:teuthology.orchestra.run.smithi066.stderr:dumped monmap epoch 1 2023-11-17T05:35:31.988 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:35:31 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:35:31.052926+0000 mon.smithi040 (mon.0) 341 : audit [DBG] from='client.? 172.21.15.66:0/2835729631' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-11-17T05:35:32.443 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2023-11-17T05:35:32.443 DEBUG:teuthology.orchestra.run.smithi066:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.4 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ed6470e2-8508-11ee-8dbe-212e2dc638e7 -- ceph mon dump -f json 2023-11-17T05:35:32.988 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:35:32 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:35:31.557060+0000 mgr.smithi040.ogcprl (mgr.14164) 441 : cluster [DBG] pgmap v425: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:35:33.674 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:33.673+0000 7ffa2520c700 1 -- 172.21.15.66:0/2504200049 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7ffa20060220 msgr2=0x7ffa200f9150 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:35:33.674 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:33.673+0000 7ffa2520c700 1 --2- 172.21.15.66:0/2504200049 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7ffa20060220 0x7ffa200f9150 secure :-1 s=READY pgs=447 cs=0 l=1 rev1=1 rx=0x7ffa10009fb0 tx=0x7ffa10009b30).stop 2023-11-17T05:35:33.674 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:33.673+0000 7ffa2520c700 1 -- 172.21.15.66:0/2504200049 shutdown_connections 2023-11-17T05:35:33.674 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:33.673+0000 7ffa2520c700 1 --2- 172.21.15.66:0/2504200049 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7ffa20060220 0x7ffa200f9150 unknown :-1 s=CLOSED pgs=447 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:35:33.675 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:33.673+0000 7ffa2520c700 1 -- 172.21.15.66:0/2504200049 >> 172.21.15.66:0/2504200049 conn(0x7ffa200f0b70 msgr2=0x7ffa200f2f90 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:35:33.675 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:33.673+0000 7ffa2520c700 1 -- 172.21.15.66:0/2504200049 shutdown_connections 2023-11-17T05:35:33.675 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:33.673+0000 7ffa2520c700 1 -- 172.21.15.66:0/2504200049 wait complete. 2023-11-17T05:35:33.676 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:33.673+0000 7ffa2520c700 1 Processor -- start 2023-11-17T05:35:33.677 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:33.674+0000 7ffa2520c700 1 -- start start 2023-11-17T05:35:33.677 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:33.674+0000 7ffa2520c700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7ffa20060220 0x7ffa200f6bd0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:35:33.677 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:33.674+0000 7ffa2520c700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7ffa200f5050 con 0x7ffa20060220 2023-11-17T05:35:33.677 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:33.674+0000 7ffa1ed9d700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7ffa20060220 0x7ffa200f6bd0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:35:33.677 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:33.674+0000 7ffa1ed9d700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7ffa20060220 0x7ffa200f6bd0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.66:45998/0 (socket says 172.21.15.66:45998) 2023-11-17T05:35:33.677 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:33.674+0000 7ffa1ed9d700 1 -- 172.21.15.66:0/1285584718 learned_addr learned my addr 172.21.15.66:0/1285584718 (peer_addr_for_me v2:172.21.15.66:0/0) 2023-11-17T05:35:33.677 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:33.674+0000 7ffa1ed9d700 1 -- 172.21.15.66:0/1285584718 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7ffa10005040 con 0x7ffa20060220 2023-11-17T05:35:33.677 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:33.674+0000 7ffa1ed9d700 1 --2- 172.21.15.66:0/1285584718 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7ffa20060220 0x7ffa200f6bd0 secure :-1 s=READY pgs=448 cs=0 l=1 rev1=1 rx=0x7ffa10013510 tx=0x7ffa1000b040).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:35:33.677 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:33.674+0000 7ffa077fe700 1 -- 172.21.15.66:0/1285584718 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7ffa10009320 con 0x7ffa20060220 2023-11-17T05:35:33.677 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:33.675+0000 7ffa2520c700 1 -- 172.21.15.66:0/1285584718 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7ffa200f51f0 con 0x7ffa20060220 2023-11-17T05:35:33.677 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:33.675+0000 7ffa077fe700 1 -- 172.21.15.66:0/1285584718 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(25 keys) v1 ==== 1074+0+0 (secure 0 0 0) 0x7ffa10013f00 con 0x7ffa20060220 2023-11-17T05:35:33.677 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:33.675+0000 7ffa077fe700 1 -- 172.21.15.66:0/1285584718 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7ffa10014dc0 con 0x7ffa20060220 2023-11-17T05:35:33.677 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:33.675+0000 7ffa2520c700 1 -- 172.21.15.66:0/1285584718 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7ffa200f5680 con 0x7ffa20060220 2023-11-17T05:35:33.678 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:33.675+0000 7ffa077fe700 1 -- 172.21.15.66:0/1285584718 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 17) v1 ==== 36269+0+0 (secure 0 0 0) 0x7ffa10014420 con 0x7ffa20060220 2023-11-17T05:35:33.678 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:33.675+0000 7ffa2520c700 1 -- 172.21.15.66:0/1285584718 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7ffa20059100 con 0x7ffa20060220 2023-11-17T05:35:33.678 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:33.675+0000 7ffa077fe700 1 --2- 172.21.15.66:0/1285584718 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7ffa0802ff60 0x7ffa08032420 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:35:33.678 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:33.676+0000 7ffa077fe700 1 -- 172.21.15.66:0/1285584718 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(5..5 src has 1..5) v4 ==== 1270+0+0 (secure 0 0 0) 0x7ffa10041050 con 0x7ffa20060220 2023-11-17T05:35:33.680 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:33.676+0000 7ffa1e59c700 1 --2- 172.21.15.66:0/1285584718 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7ffa0802ff60 0x7ffa08032420 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:35:33.680 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:33.676+0000 7ffa1e59c700 1 --2- 172.21.15.66:0/1285584718 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7ffa0802ff60 0x7ffa08032420 secure :-1 s=READY pgs=189 cs=0 l=1 rev1=1 rx=0x7ffa140096a0 tx=0x7ffa14006b40).ready entity=mgr.14164 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:35:33.681 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:33.680+0000 7ffa077fe700 1 -- 172.21.15.66:0/1285584718 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+152212 (secure 0 0 0) 0x7ffa100095f0 con 0x7ffa20060220 2023-11-17T05:35:33.899 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:33.898+0000 7ffa2520c700 1 -- 172.21.15.66:0/1285584718 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "mon dump", "format": "json"} v 0) v1 -- 0x7ffa200fbf50 con 0x7ffa20060220 2023-11-17T05:35:33.900 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:33.899+0000 7ffa077fe700 1 -- 172.21.15.66:0/1285584718 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mon_command_ack([{"prefix": "mon dump", "format": "json"}]=0 dumped monmap epoch 1 v1) v1 ==== 95+0+690 (secure 0 0 0) 0x7ffa1001cb10 con 0x7ffa20060220 2023-11-17T05:35:33.900 INFO:teuthology.orchestra.run.smithi066.stdout: 2023-11-17T05:35:33.900 INFO:teuthology.orchestra.run.smithi066.stdout:{"epoch":1,"fsid":"ed6470e2-8508-11ee-8dbe-212e2dc638e7","modified":"2023-11-17T05:20:13.934038Z","created":"2023-11-17T05:20:13.934038Z","min_mon_release":16,"min_mon_release_name":"pacific","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging"],"optional":[]},"mons":[{"rank":0,"name":"smithi040","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:3300","nonce":0},{"type":"v1","addr":"172.21.15.40:6789","nonce":0}]},"addr":"172.21.15.40:6789/0","public_addr":"172.21.15.40:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2023-11-17T05:35:33.901 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:33.900+0000 7ffa2520c700 1 -- 172.21.15.66:0/1285584718 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7ffa0802ff60 msgr2=0x7ffa08032420 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:35:33.902 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:33.900+0000 7ffa2520c700 1 --2- 172.21.15.66:0/1285584718 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7ffa0802ff60 0x7ffa08032420 secure :-1 s=READY pgs=189 cs=0 l=1 rev1=1 rx=0x7ffa140096a0 tx=0x7ffa14006b40).stop 2023-11-17T05:35:33.902 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:33.900+0000 7ffa2520c700 1 -- 172.21.15.66:0/1285584718 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7ffa20060220 msgr2=0x7ffa200f6bd0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:35:33.902 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:33.900+0000 7ffa2520c700 1 --2- 172.21.15.66:0/1285584718 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7ffa20060220 0x7ffa200f6bd0 secure :-1 s=READY pgs=448 cs=0 l=1 rev1=1 rx=0x7ffa10013510 tx=0x7ffa1000b040).stop 2023-11-17T05:35:33.902 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:33.900+0000 7ffa2520c700 1 -- 172.21.15.66:0/1285584718 shutdown_connections 2023-11-17T05:35:33.902 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:33.900+0000 7ffa2520c700 1 --2- 172.21.15.66:0/1285584718 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7ffa0802ff60 0x7ffa08032420 unknown :-1 s=CLOSED pgs=189 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:35:33.902 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:33.900+0000 7ffa2520c700 1 --2- 172.21.15.66:0/1285584718 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7ffa20060220 0x7ffa200f6bd0 unknown :-1 s=CLOSED pgs=448 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:35:33.902 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:33.900+0000 7ffa2520c700 1 -- 172.21.15.66:0/1285584718 >> 172.21.15.66:0/1285584718 conn(0x7ffa200f0b70 msgr2=0x7ffa200f1660 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:35:33.902 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:33.900+0000 7ffa2520c700 1 -- 172.21.15.66:0/1285584718 shutdown_connections 2023-11-17T05:35:33.902 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:33.900+0000 7ffa2520c700 1 -- 172.21.15.66:0/1285584718 wait complete. 2023-11-17T05:35:33.903 INFO:teuthology.orchestra.run.smithi066.stderr:dumped monmap epoch 1 2023-11-17T05:35:33.989 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:35:33 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:35:33.557475+0000 mgr.smithi040.ogcprl (mgr.14164) 442 : cluster [DBG] pgmap v426: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:35:34.989 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:35:34 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:35:33.900292+0000 mon.smithi040 (mon.0) 342 : audit [DBG] from='client.? 172.21.15.66:0/1285584718' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-11-17T05:35:35.284 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2023-11-17T05:35:35.284 DEBUG:teuthology.orchestra.run.smithi066:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.4 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ed6470e2-8508-11ee-8dbe-212e2dc638e7 -- ceph mon dump -f json 2023-11-17T05:35:35.989 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:35:35 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:35:35.557796+0000 mgr.smithi040.ogcprl (mgr.14164) 443 : cluster [DBG] pgmap v427: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:35:36.459 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:36.451+0000 7fb58ee97700 1 -- 172.21.15.66:0/1572502096 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fb5880f51a0 msgr2=0x7fb5880f55c0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:35:36.459 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:36.451+0000 7fb58ee97700 1 --2- 172.21.15.66:0/1572502096 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fb5880f51a0 0x7fb5880f55c0 secure :-1 s=READY pgs=449 cs=0 l=1 rev1=1 rx=0x7fb578009fb0 tx=0x7fb578009b30).stop 2023-11-17T05:35:36.459 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:36.451+0000 7fb58ee97700 1 -- 172.21.15.66:0/1572502096 shutdown_connections 2023-11-17T05:35:36.460 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:36.451+0000 7fb58ee97700 1 --2- 172.21.15.66:0/1572502096 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fb5880f51a0 0x7fb5880f55c0 unknown :-1 s=CLOSED pgs=449 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:35:36.460 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:36.451+0000 7fb58ee97700 1 -- 172.21.15.66:0/1572502096 >> 172.21.15.66:0/1572502096 conn(0x7fb5880f09a0 msgr2=0x7fb5880f2dc0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:35:36.460 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:36.451+0000 7fb58ee97700 1 -- 172.21.15.66:0/1572502096 shutdown_connections 2023-11-17T05:35:36.460 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:36.451+0000 7fb58ee97700 1 -- 172.21.15.66:0/1572502096 wait complete. 2023-11-17T05:35:36.460 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:36.452+0000 7fb58ee97700 1 Processor -- start 2023-11-17T05:35:36.460 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:36.452+0000 7fb58ee97700 1 -- start start 2023-11-17T05:35:36.460 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:36.452+0000 7fb58ee97700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fb5880f51a0 0x7fb5881053b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:35:36.460 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:36.452+0000 7fb58ee97700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fb5881058a0 con 0x7fb5880f51a0 2023-11-17T05:35:36.460 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:36.452+0000 7fb58de95700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fb5880f51a0 0x7fb5881053b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:35:36.460 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:36.452+0000 7fb58de95700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fb5880f51a0 0x7fb5881053b0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.66:46016/0 (socket says 172.21.15.66:46016) 2023-11-17T05:35:36.460 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:36.452+0000 7fb58de95700 1 -- 172.21.15.66:0/1312558794 learned_addr learned my addr 172.21.15.66:0/1312558794 (peer_addr_for_me v2:172.21.15.66:0/0) 2023-11-17T05:35:36.461 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:36.453+0000 7fb58de95700 1 -- 172.21.15.66:0/1312558794 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fb578005040 con 0x7fb5880f51a0 2023-11-17T05:35:36.461 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:36.453+0000 7fb58de95700 1 --2- 172.21.15.66:0/1312558794 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fb5880f51a0 0x7fb5881053b0 secure :-1 s=READY pgs=450 cs=0 l=1 rev1=1 rx=0x7fb578013510 tx=0x7fb57800b040).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:35:36.461 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:36.453+0000 7fb57e7fc700 1 -- 172.21.15.66:0/1312558794 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7fb578009320 con 0x7fb5880f51a0 2023-11-17T05:35:36.461 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:36.453+0000 7fb58ee97700 1 -- 172.21.15.66:0/1312558794 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fb588105aa0 con 0x7fb5880f51a0 2023-11-17T05:35:36.461 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:36.453+0000 7fb57e7fc700 1 -- 172.21.15.66:0/1312558794 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(25 keys) v1 ==== 1074+0+0 (secure 0 0 0) 0x7fb578013f00 con 0x7fb5880f51a0 2023-11-17T05:35:36.461 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:36.453+0000 7fb57e7fc700 1 -- 172.21.15.66:0/1312558794 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 229+0+0 (secure 0 0 0) 0x7fb578014dc0 con 0x7fb5880f51a0 2023-11-17T05:35:36.461 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:36.453+0000 7fb58ee97700 1 -- 172.21.15.66:0/1312558794 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fb588105f30 con 0x7fb5880f51a0 2023-11-17T05:35:37.016 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:36.454+0000 7fb57e7fc700 1 -- 172.21.15.66:0/1312558794 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 17) v1 ==== 36269+0+0 (secure 0 0 0) 0x7fb5780095f0 con 0x7fb5880f51a0 2023-11-17T05:35:37.016 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:36.454+0000 7fb58ee97700 1 -- 172.21.15.66:0/1312558794 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fb588045ad0 con 0x7fb5880f51a0 2023-11-17T05:35:37.016 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:36.454+0000 7fb57e7fc700 1 --2- 172.21.15.66:0/1312558794 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fb57402ff60 0x7fb574032420 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:35:37.016 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:36.454+0000 7fb57e7fc700 1 -- 172.21.15.66:0/1312558794 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(5..5 src has 1..5) v4 ==== 1270+0+0 (secure 0 0 0) 0x7fb578041000 con 0x7fb5880f51a0 2023-11-17T05:35:37.017 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:36.454+0000 7fb58d694700 1 --2- 172.21.15.66:0/1312558794 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fb57402ff60 0x7fb574032420 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:35:37.017 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:36.455+0000 7fb58d694700 1 --2- 172.21.15.66:0/1312558794 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fb57402ff60 0x7fb574032420 secure :-1 s=READY pgs=190 cs=0 l=1 rev1=1 rx=0x7fb5840096a0 tx=0x7fb584006b40).ready entity=mgr.14164 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:35:37.019 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:35:36 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:35:36.678384+0000 mon.smithi040 (mon.0) 343 : audit [DBG] from='client.? 172.21.15.66:0/1312558794' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-11-17T05:35:37.019 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:36.459+0000 7fb57e7fc700 1 -- 172.21.15.66:0/1312558794 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+152212 (secure 0 0 0) 0x7fb57801c790 con 0x7fb5880f51a0 2023-11-17T05:35:37.071 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:36.676+0000 7fb58ee97700 1 -- 172.21.15.66:0/1312558794 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "mon dump", "format": "json"} v 0) v1 -- 0x7fb588106d80 con 0x7fb5880f51a0 2023-11-17T05:35:37.072 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:36.677+0000 7fb57e7fc700 1 -- 172.21.15.66:0/1312558794 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mon_command_ack([{"prefix": "mon dump", "format": "json"}]=0 dumped monmap epoch 1 v1) v1 ==== 95+0+690 (secure 0 0 0) 0x7fb57801c790 con 0x7fb5880f51a0 2023-11-17T05:35:37.072 INFO:teuthology.orchestra.run.smithi066.stdout: 2023-11-17T05:35:37.072 INFO:teuthology.orchestra.run.smithi066.stdout:{"epoch":1,"fsid":"ed6470e2-8508-11ee-8dbe-212e2dc638e7","modified":"2023-11-17T05:20:13.934038Z","created":"2023-11-17T05:20:13.934038Z","min_mon_release":16,"min_mon_release_name":"pacific","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging"],"optional":[]},"mons":[{"rank":0,"name":"smithi040","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:3300","nonce":0},{"type":"v1","addr":"172.21.15.40:6789","nonce":0}]},"addr":"172.21.15.40:6789/0","public_addr":"172.21.15.40:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2023-11-17T05:35:37.073 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:36.678+0000 7fb58ee97700 1 -- 172.21.15.66:0/1312558794 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fb57402ff60 msgr2=0x7fb574032420 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:35:37.073 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:36.678+0000 7fb58ee97700 1 --2- 172.21.15.66:0/1312558794 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fb57402ff60 0x7fb574032420 secure :-1 s=READY pgs=190 cs=0 l=1 rev1=1 rx=0x7fb5840096a0 tx=0x7fb584006b40).stop 2023-11-17T05:35:37.073 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:36.678+0000 7fb58ee97700 1 -- 172.21.15.66:0/1312558794 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fb5880f51a0 msgr2=0x7fb5881053b0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:35:37.073 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:36.678+0000 7fb58ee97700 1 --2- 172.21.15.66:0/1312558794 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fb5880f51a0 0x7fb5881053b0 secure :-1 s=READY pgs=450 cs=0 l=1 rev1=1 rx=0x7fb578013510 tx=0x7fb57800b040).stop 2023-11-17T05:35:37.073 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:36.678+0000 7fb58ee97700 1 -- 172.21.15.66:0/1312558794 shutdown_connections 2023-11-17T05:35:37.073 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:36.678+0000 7fb58ee97700 1 --2- 172.21.15.66:0/1312558794 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fb57402ff60 0x7fb574032420 unknown :-1 s=CLOSED pgs=190 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:35:37.073 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:36.678+0000 7fb58ee97700 1 --2- 172.21.15.66:0/1312558794 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fb5880f51a0 0x7fb5881053b0 unknown :-1 s=CLOSED pgs=450 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:35:37.073 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:36.678+0000 7fb58ee97700 1 -- 172.21.15.66:0/1312558794 >> 172.21.15.66:0/1312558794 conn(0x7fb5880f09a0 msgr2=0x7fb5880f1460 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:35:37.073 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:36.678+0000 7fb58ee97700 1 -- 172.21.15.66:0/1312558794 shutdown_connections 2023-11-17T05:35:37.073 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:35:36.678+0000 7fb58ee97700 1 -- 172.21.15.66:0/1312558794 wait complete. 2023-11-17T05:35:37.073 INFO:teuthology.orchestra.run.smithi066.stderr:dumped monmap epoch 1 2023-11-17T05:35:37.988 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:35:37 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:35:37.558066+0000 mgr.smithi040.ogcprl (mgr.14164) 444 : cluster [DBG] pgmap v428: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:36:24.939 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2023-11-17T05:36:24.961 DEBUG:teuthology.orchestra.run.smithi066:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.4 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ed6470e2-8508-11ee-8dbe-212e2dc638e7 -- ceph mon dump -f json 2023-11-17T05:36:24.967 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 systemd[1]: Starting Ceph mon.smithi066 for ed6470e2-8508-11ee-8dbe-212e2dc638e7... 2023-11-17T05:36:24.968 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: debug 2023-11-17T05:36:14.547+0000 7f70ffa73700 0 set uid:gid to 167:167 (ceph:ceph) 2023-11-17T05:36:24.968 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: debug 2023-11-17T05:36:14.547+0000 7f70ffa73700 0 ceph version 16.2.4 (3cbe25cde3cfa028984618ad32de9edc4c1eaed0) pacific (stable), process ceph-mon, pid 7 2023-11-17T05:36:24.968 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: debug 2023-11-17T05:36:14.547+0000 7f70ffa73700 0 pidfile_write: ignore empty --pid-file 2023-11-17T05:36:24.968 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: debug 2023-11-17T05:36:14.551+0000 7f70ffa73700 0 load: jerasure load: lrc load: isa 2023-11-17T05:36:25.013 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: debug 2023-11-17T05:36:14.553+0000 7f70ffa73700 4 rocksdb: RocksDB version: 6.8.1 2023-11-17T05:36:25.014 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: 2023-11-17T05:36:25.014 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: debug 2023-11-17T05:36:14.553+0000 7f70ffa73700 4 rocksdb: Git sha rocksdb_build_git_sha:@0@ 2023-11-17T05:36:25.014 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: debug 2023-11-17T05:36:14.553+0000 7f70ffa73700 4 rocksdb: Compile date May 13 2021 2023-11-17T05:36:25.014 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: debug 2023-11-17T05:36:14.553+0000 7f70ffa73700 4 rocksdb: DB SUMMARY 2023-11-17T05:36:25.014 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: 2023-11-17T05:36:25.014 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: debug 2023-11-17T05:36:14.553+0000 7f70ffa73700 4 rocksdb: CURRENT file: CURRENT 2023-11-17T05:36:25.014 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: 2023-11-17T05:36:25.014 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: debug 2023-11-17T05:36:14.553+0000 7f70ffa73700 4 rocksdb: IDENTITY file: IDENTITY 2023-11-17T05:36:25.014 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: 2023-11-17T05:36:25.014 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: debug 2023-11-17T05:36:14.553+0000 7f70ffa73700 4 rocksdb: MANIFEST file: MANIFEST-000001 size: 13 Bytes 2023-11-17T05:36:25.014 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: 2023-11-17T05:36:25.014 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: debug 2023-11-17T05:36:14.553+0000 7f70ffa73700 4 rocksdb: SST files in /var/lib/ceph/mon/ceph-smithi066/store.db dir, Total Num: 0, files: 2023-11-17T05:36:25.015 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: 2023-11-17T05:36:25.015 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: debug 2023-11-17T05:36:14.553+0000 7f70ffa73700 4 rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-smithi066/store.db: 000003.log size: 511 ; 2023-11-17T05:36:25.015 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: 2023-11-17T05:36:25.015 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: debug 2023-11-17T05:36:14.553+0000 7f70ffa73700 4 rocksdb: Options.error_if_exists: 0 2023-11-17T05:36:25.015 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: debug 2023-11-17T05:36:14.553+0000 7f70ffa73700 4 rocksdb: Options.create_if_missing: 0 2023-11-17T05:36:25.015 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: debug 2023-11-17T05:36:14.553+0000 7f70ffa73700 4 rocksdb: Options.paranoid_checks: 1 2023-11-17T05:36:25.015 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: debug 2023-11-17T05:36:14.553+0000 7f70ffa73700 4 rocksdb: Options.env: 0x5631a54f61c0 2023-11-17T05:36:25.015 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: debug 2023-11-17T05:36:14.553+0000 7f70ffa73700 4 rocksdb: Options.fs: Posix File System 2023-11-17T05:36:25.015 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: debug 2023-11-17T05:36:14.553+0000 7f70ffa73700 4 rocksdb: Options.info_log: 0x5631a6bcf960 2023-11-17T05:36:25.015 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: debug 2023-11-17T05:36:14.553+0000 7f70ffa73700 4 rocksdb: Options.max_file_opening_threads: 16 2023-11-17T05:36:25.015 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: debug 2023-11-17T05:36:14.553+0000 7f70ffa73700 4 rocksdb: Options.statistics: (nil) 2023-11-17T05:36:25.015 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: debug 2023-11-17T05:36:14.553+0000 7f70ffa73700 4 rocksdb: Options.use_fsync: 0 2023-11-17T05:36:25.015 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: debug 2023-11-17T05:36:14.553+0000 7f70ffa73700 4 rocksdb: Options.max_log_file_size: 0 2023-11-17T05:36:25.016 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: debug 2023-11-17T05:36:14.553+0000 7f70ffa73700 4 rocksdb: Options.max_manifest_file_size: 1073741824 2023-11-17T05:36:25.016 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: debug 2023-11-17T05:36:14.553+0000 7f70ffa73700 4 rocksdb: Options.log_file_time_to_roll: 0 2023-11-17T05:36:25.053 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: debug 2023-11-17T05:36:14.553+0000 7f70ffa73700 4 rocksdb: Options.keep_log_file_num: 1000 2023-11-17T05:36:25.154 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: debug 2023-11-17T05:36:14.553+0000 7f70ffa73700 4 rocksdb: Options.recycle_log_file_num: 0 2023-11-17T05:36:25.154 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: debug 2023-11-17T05:36:14.553+0000 7f70ffa73700 4 rocksdb: Options.allow_fallocate: 1 2023-11-17T05:36:25.154 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: debug 2023-11-17T05:36:14.553+0000 7f70ffa73700 4 rocksdb: Options.allow_mmap_reads: 0 2023-11-17T05:36:25.154 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: debug 2023-11-17T05:36:14.553+0000 7f70ffa73700 4 rocksdb: Options.allow_mmap_writes: 0 2023-11-17T05:36:25.154 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: debug 2023-11-17T05:36:14.553+0000 7f70ffa73700 4 rocksdb: Options.use_direct_reads: 0 2023-11-17T05:36:25.155 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: debug 2023-11-17T05:36:14.553+0000 7f70ffa73700 4 rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2023-11-17T05:36:25.155 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: debug 2023-11-17T05:36:14.553+0000 7f70ffa73700 4 rocksdb: Options.create_missing_column_families: 0 2023-11-17T05:36:25.155 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: debug 2023-11-17T05:36:14.553+0000 7f70ffa73700 4 rocksdb: Options.db_log_dir: 2023-11-17T05:36:25.155 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: debug 2023-11-17T05:36:14.553+0000 7f70ffa73700 4 rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-smithi066/store.db 2023-11-17T05:36:25.155 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: debug 2023-11-17T05:36:14.553+0000 7f70ffa73700 4 rocksdb: Options.table_cache_numshardbits: 6 2023-11-17T05:36:25.155 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: debug 2023-11-17T05:36:14.553+0000 7f70ffa73700 4 rocksdb: Options.max_subcompactions: 1 2023-11-17T05:36:25.155 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: debug 2023-11-17T05:36:14.553+0000 7f70ffa73700 4 rocksdb: Options.max_background_flushes: -1 2023-11-17T05:36:25.155 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: debug 2023-11-17T05:36:14.553+0000 7f70ffa73700 4 rocksdb: Options.WAL_ttl_seconds: 0 2023-11-17T05:36:25.155 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: debug 2023-11-17T05:36:14.553+0000 7f70ffa73700 4 rocksdb: Options.WAL_size_limit_MB: 0 2023-11-17T05:36:25.155 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: debug 2023-11-17T05:36:14.553+0000 7f70ffa73700 4 rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2023-11-17T05:36:25.155 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: debug 2023-11-17T05:36:14.553+0000 7f70ffa73700 4 rocksdb: Options.manifest_preallocation_size: 4194304 2023-11-17T05:36:25.155 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: debug 2023-11-17T05:36:14.553+0000 7f70ffa73700 4 rocksdb: Options.is_fd_close_on_exec: 1 2023-11-17T05:36:25.177 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: debug 2023-11-17T05:36:14.553+0000 7f70ffa73700 4 rocksdb: Options.advise_random_on_open: 1 2023-11-17T05:36:25.177 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: debug 2023-11-17T05:36:14.553+0000 7f70ffa73700 4 rocksdb: Options.db_write_buffer_size: 0 2023-11-17T05:36:25.177 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: debug 2023-11-17T05:36:14.553+0000 7f70ffa73700 4 rocksdb: Options.write_buffer_manager: 0x5631a7844030 2023-11-17T05:36:25.177 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: debug 2023-11-17T05:36:14.553+0000 7f70ffa73700 4 rocksdb: Options.access_hint_on_compaction_start: 1 2023-11-17T05:36:25.177 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: debug 2023-11-17T05:36:14.553+0000 7f70ffa73700 4 rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2023-11-17T05:36:25.177 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: debug 2023-11-17T05:36:14.553+0000 7f70ffa73700 4 rocksdb: Options.random_access_max_buffer_size: 1048576 2023-11-17T05:36:25.177 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: debug 2023-11-17T05:36:14.553+0000 7f70ffa73700 4 rocksdb: Options.use_adaptive_mutex: 0 2023-11-17T05:36:25.177 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: debug 2023-11-17T05:36:14.553+0000 7f70ffa73700 4 rocksdb: Options.rate_limiter: (nil) 2023-11-17T05:36:25.177 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: debug 2023-11-17T05:36:14.553+0000 7f70ffa73700 4 rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2023-11-17T05:36:25.177 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: debug 2023-11-17T05:36:14.553+0000 7f70ffa73700 4 rocksdb: Options.wal_recovery_mode: 2 2023-11-17T05:36:25.177 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: debug 2023-11-17T05:36:14.553+0000 7f70ffa73700 4 rocksdb: Options.enable_thread_tracking: 0 2023-11-17T05:36:25.177 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: debug 2023-11-17T05:36:14.553+0000 7f70ffa73700 4 rocksdb: Options.enable_pipelined_write: 0 2023-11-17T05:36:25.178 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: debug 2023-11-17T05:36:14.553+0000 7f70ffa73700 4 rocksdb: Options.unordered_write: 0 2023-11-17T05:36:25.178 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: debug 2023-11-17T05:36:14.553+0000 7f70ffa73700 4 rocksdb: Options.allow_concurrent_memtable_write: 1 2023-11-17T05:36:25.178 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: debug 2023-11-17T05:36:14.553+0000 7f70ffa73700 4 rocksdb: Options.enable_write_thread_adaptive_yield: 1 2023-11-17T05:36:25.178 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: debug 2023-11-17T05:36:14.553+0000 7f70ffa73700 4 rocksdb: Options.write_thread_max_yield_usec: 100 2023-11-17T05:36:25.178 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: debug 2023-11-17T05:36:14.553+0000 7f70ffa73700 4 rocksdb: Options.write_thread_slow_yield_usec: 3 2023-11-17T05:36:25.178 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: debug 2023-11-17T05:36:14.553+0000 7f70ffa73700 4 rocksdb: Options.row_cache: None 2023-11-17T05:36:25.178 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: debug 2023-11-17T05:36:14.553+0000 7f70ffa73700 4 rocksdb: Options.wal_filter: None 2023-11-17T05:36:25.178 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: debug 2023-11-17T05:36:14.553+0000 7f70ffa73700 4 rocksdb: Options.avoid_flush_during_recovery: 0 2023-11-17T05:36:25.178 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: debug 2023-11-17T05:36:14.553+0000 7f70ffa73700 4 rocksdb: Options.allow_ingest_behind: 0 2023-11-17T05:36:25.178 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: debug 2023-11-17T05:36:14.553+0000 7f70ffa73700 4 rocksdb: Options.preserve_deletes: 0 2023-11-17T05:36:25.178 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: debug 2023-11-17T05:36:14.553+0000 7f70ffa73700 4 rocksdb: Options.two_write_queues: 0 2023-11-17T05:36:25.178 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: debug 2023-11-17T05:36:14.553+0000 7f70ffa73700 4 rocksdb: Options.manual_wal_flush: 0 2023-11-17T05:36:25.179 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: debug 2023-11-17T05:36:14.553+0000 7f70ffa73700 4 rocksdb: Options.atomic_flush: 0 2023-11-17T05:36:25.179 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: debug 2023-11-17T05:36:14.553+0000 7f70ffa73700 4 rocksdb: Options.avoid_unnecessary_blocking_io: 0 2023-11-17T05:36:25.179 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: debug 2023-11-17T05:36:14.553+0000 7f70ffa73700 4 rocksdb: Options.persist_stats_to_disk: 0 2023-11-17T05:36:25.179 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: debug 2023-11-17T05:36:14.553+0000 7f70ffa73700 4 rocksdb: Options.write_dbid_to_manifest: 0 2023-11-17T05:36:25.179 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: debug 2023-11-17T05:36:14.553+0000 7f70ffa73700 4 rocksdb: Options.log_readahead_size: 0 2023-11-17T05:36:25.179 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: debug 2023-11-17T05:36:14.553+0000 7f70ffa73700 4 rocksdb: Options.sst_file_checksum_func: Unknown 2023-11-17T05:36:25.179 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: debug 2023-11-17T05:36:14.553+0000 7f70ffa73700 4 rocksdb: Options.max_background_jobs: 2 2023-11-17T05:36:25.179 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: debug 2023-11-17T05:36:14.553+0000 7f70ffa73700 4 rocksdb: Options.max_background_compactions: -1 2023-11-17T05:36:25.179 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: debug 2023-11-17T05:36:14.553+0000 7f70ffa73700 4 rocksdb: Options.avoid_flush_during_shutdown: 0 2023-11-17T05:36:25.179 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: debug 2023-11-17T05:36:14.553+0000 7f70ffa73700 4 rocksdb: Options.writable_file_max_buffer_size: 1048576 2023-11-17T05:36:25.179 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: debug 2023-11-17T05:36:14.553+0000 7f70ffa73700 4 rocksdb: Options.delayed_write_rate : 16777216 2023-11-17T05:36:25.179 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: debug 2023-11-17T05:36:14.553+0000 7f70ffa73700 4 rocksdb: Options.max_total_wal_size: 0 2023-11-17T05:36:25.179 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: debug 2023-11-17T05:36:14.553+0000 7f70ffa73700 4 rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2023-11-17T05:36:25.180 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: debug 2023-11-17T05:36:14.553+0000 7f70ffa73700 4 rocksdb: Options.stats_dump_period_sec: 600 2023-11-17T05:36:25.180 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: debug 2023-11-17T05:36:14.553+0000 7f70ffa73700 4 rocksdb: Options.stats_persist_period_sec: 600 2023-11-17T05:36:25.180 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: debug 2023-11-17T05:36:14.553+0000 7f70ffa73700 4 rocksdb: Options.stats_history_buffer_size: 1048576 2023-11-17T05:36:25.180 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: debug 2023-11-17T05:36:14.553+0000 7f70ffa73700 4 rocksdb: Options.max_open_files: -1 2023-11-17T05:36:25.180 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: debug 2023-11-17T05:36:14.553+0000 7f70ffa73700 4 rocksdb: Options.bytes_per_sync: 0 2023-11-17T05:36:25.180 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: debug 2023-11-17T05:36:14.553+0000 7f70ffa73700 4 rocksdb: Options.wal_bytes_per_sync: 0 2023-11-17T05:36:25.180 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: debug 2023-11-17T05:36:14.553+0000 7f70ffa73700 4 rocksdb: Options.strict_bytes_per_sync: 0 2023-11-17T05:36:25.180 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: debug 2023-11-17T05:36:14.553+0000 7f70ffa73700 4 rocksdb: Options.compaction_readahead_size: 0 2023-11-17T05:36:25.180 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: debug 2023-11-17T05:36:14.553+0000 7f70ffa73700 4 rocksdb: Compression algorithms supported: 2023-11-17T05:36:25.180 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: debug 2023-11-17T05:36:14.553+0000 7f70ffa73700 4 rocksdb: kZSTDNotFinalCompression supported: 0 2023-11-17T05:36:25.180 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: debug 2023-11-17T05:36:14.553+0000 7f70ffa73700 4 rocksdb: kZSTD supported: 0 2023-11-17T05:36:25.180 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: debug 2023-11-17T05:36:14.553+0000 7f70ffa73700 4 rocksdb: kXpressCompression supported: 0 2023-11-17T05:36:25.180 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: debug 2023-11-17T05:36:14.553+0000 7f70ffa73700 4 rocksdb: kLZ4HCCompression supported: 1 2023-11-17T05:36:25.180 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: debug 2023-11-17T05:36:14.553+0000 7f70ffa73700 4 rocksdb: kLZ4Compression supported: 1 2023-11-17T05:36:25.181 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: debug 2023-11-17T05:36:14.553+0000 7f70ffa73700 4 rocksdb: kBZip2Compression supported: 0 2023-11-17T05:36:25.181 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: debug 2023-11-17T05:36:14.553+0000 7f70ffa73700 4 rocksdb: kZlibCompression supported: 1 2023-11-17T05:36:25.181 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: debug 2023-11-17T05:36:14.553+0000 7f70ffa73700 4 rocksdb: kSnappyCompression supported: 1 2023-11-17T05:36:25.266 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: debug 2023-11-17T05:36:14.553+0000 7f70ffa73700 4 rocksdb: Fast CRC32 supported: Supported on x86 2023-11-17T05:36:25.266 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: debug 2023-11-17T05:36:14.553+0000 7f70ffa73700 4 rocksdb: [version_set.cc:4413] Recovering from manifest file: /var/lib/ceph/mon/ceph-smithi066/store.db/MANIFEST-000001 2023-11-17T05:36:25.266 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: 2023-11-17T05:36:25.266 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 bash[145361]: 88e4d3f36f4cc704397345734f6b4488b4a57cf3b1a081cfaddbd98c4cff2b4a 2023-11-17T05:36:25.267 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: debug 2023-11-17T05:36:14.553+0000 7f70ffa73700 4 rocksdb: [column_family.cc:552] --------------- Options for column family [default]: 2023-11-17T05:36:25.267 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: 2023-11-17T05:36:25.267 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: debug 2023-11-17T05:36:14.553+0000 7f70ffa73700 4 rocksdb: Options.comparator: leveldb.BytewiseComparator 2023-11-17T05:36:25.267 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: debug 2023-11-17T05:36:14.553+0000 7f70ffa73700 4 rocksdb: Options.merge_operator: 2023-11-17T05:36:25.268 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: debug 2023-11-17T05:36:14.553+0000 7f70ffa73700 4 rocksdb: Options.compaction_filter: None 2023-11-17T05:36:25.268 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: debug 2023-11-17T05:36:14.553+0000 7f70ffa73700 4 rocksdb: Options.compaction_filter_factory: None 2023-11-17T05:36:25.268 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: debug 2023-11-17T05:36:14.553+0000 7f70ffa73700 4 rocksdb: Options.memtable_factory: SkipListFactory 2023-11-17T05:36:25.268 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: debug 2023-11-17T05:36:14.553+0000 7f70ffa73700 4 rocksdb: Options.table_factory: BlockBasedTable 2023-11-17T05:36:25.268 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: debug 2023-11-17T05:36:14.553+0000 7f70ffa73700 4 rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x5631a6ae6128) 2023-11-17T05:36:25.268 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: cache_index_and_filter_blocks: 1 2023-11-17T05:36:25.268 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: cache_index_and_filter_blocks_with_high_priority: 0 2023-11-17T05:36:25.268 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: pin_l0_filter_and_index_blocks_in_cache: 0 2023-11-17T05:36:25.268 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: pin_top_level_index_and_filter: 1 2023-11-17T05:36:25.268 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: index_type: 0 2023-11-17T05:36:25.268 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: data_block_index_type: 0 2023-11-17T05:36:25.269 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: index_shortening: 1 2023-11-17T05:36:25.269 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: data_block_hash_table_util_ratio: 0.750000 2023-11-17T05:36:25.269 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: hash_index_allow_collision: 1 2023-11-17T05:36:25.269 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: checksum: 1 2023-11-17T05:36:25.269 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: no_block_cache: 0 2023-11-17T05:36:25.269 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: block_cache: 0x5631a6b1cf10 2023-11-17T05:36:25.269 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: block_cache_name: BinnedLRUCache 2023-11-17T05:36:25.269 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: block_cache_options: 2023-11-17T05:36:25.269 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: capacity : 536870912 2023-11-17T05:36:25.269 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: num_shard_bits : 4 2023-11-17T05:36:25.269 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: strict_capacity_limit : 0 2023-11-17T05:36:25.269 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: high_pri_pool_ratio: 0.000 2023-11-17T05:36:25.270 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: block_cache_compressed: (nil) 2023-11-17T05:36:25.270 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: persistent_cache: (nil) 2023-11-17T05:36:25.270 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: block_size: 4096 2023-11-17T05:36:25.270 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: block_size_deviation: 10 2023-11-17T05:36:25.270 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: block_restart_interval: 16 2023-11-17T05:36:25.270 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: index_block_restart_interval: 1 2023-11-17T05:36:25.270 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: metadata_block_size: 4096 2023-11-17T05:36:25.713 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: partition_filters: 0 2023-11-17T05:36:25.713 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: use_delta_encoding: 1 2023-11-17T05:36:25.713 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: filter_policy: rocksdb.BuiltinBloomFilter 2023-11-17T05:36:25.713 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: whole_key_filtering: 1 2023-11-17T05:36:25.714 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: verify_compression: 0 2023-11-17T05:36:25.714 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: read_amp_bytes_per_bit: 0 2023-11-17T05:36:25.714 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: format_version: 2 2023-11-17T05:36:25.714 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: enable_index_compression: 1 2023-11-17T05:36:25.714 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: block_align: 0 2023-11-17T05:36:25.714 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: 2023-11-17T05:36:25.714 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: debug 2023-11-17T05:36:14.553+0000 7f70ffa73700 4 rocksdb: Options.write_buffer_size: 33554432 2023-11-17T05:36:25.714 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: debug 2023-11-17T05:36:14.553+0000 7f70ffa73700 4 rocksdb: Options.max_write_buffer_number: 2 2023-11-17T05:36:25.714 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: debug 2023-11-17T05:36:14.553+0000 7f70ffa73700 4 rocksdb: Options.compression: NoCompression 2023-11-17T05:36:25.714 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: debug 2023-11-17T05:36:14.553+0000 7f70ffa73700 4 rocksdb: Options.bottommost_compression: Disabled 2023-11-17T05:36:25.714 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: debug 2023-11-17T05:36:14.553+0000 7f70ffa73700 4 rocksdb: Options.prefix_extractor: nullptr 2023-11-17T05:36:25.714 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: debug 2023-11-17T05:36:14.553+0000 7f70ffa73700 4 rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2023-11-17T05:36:25.715 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: debug 2023-11-17T05:36:14.553+0000 7f70ffa73700 4 rocksdb: Options.num_levels: 7 2023-11-17T05:36:25.715 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: debug 2023-11-17T05:36:14.553+0000 7f70ffa73700 4 rocksdb: Options.min_write_buffer_number_to_merge: 1 2023-11-17T05:36:25.715 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: debug 2023-11-17T05:36:14.553+0000 7f70ffa73700 4 rocksdb: Options.max_write_buffer_number_to_maintain: 0 2023-11-17T05:36:25.715 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: debug 2023-11-17T05:36:14.553+0000 7f70ffa73700 4 rocksdb: Options.max_write_buffer_size_to_maintain: 0 2023-11-17T05:36:25.715 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: debug 2023-11-17T05:36:14.553+0000 7f70ffa73700 4 rocksdb: Options.bottommost_compression_opts.window_bits: -14 2023-11-17T05:36:25.715 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: debug 2023-11-17T05:36:14.553+0000 7f70ffa73700 4 rocksdb: Options.bottommost_compression_opts.level: 32767 2023-11-17T05:36:25.715 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: debug 2023-11-17T05:36:14.553+0000 7f70ffa73700 4 rocksdb: Options.bottommost_compression_opts.strategy: 0 2023-11-17T05:36:25.715 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: debug 2023-11-17T05:36:14.553+0000 7f70ffa73700 4 rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2023-11-17T05:36:25.715 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: debug 2023-11-17T05:36:14.553+0000 7f70ffa73700 4 rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2023-11-17T05:36:25.715 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: debug 2023-11-17T05:36:14.553+0000 7f70ffa73700 4 rocksdb: Options.bottommost_compression_opts.enabled: false 2023-11-17T05:36:25.715 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: debug 2023-11-17T05:36:14.553+0000 7f70ffa73700 4 rocksdb: Options.compression_opts.window_bits: -14 2023-11-17T05:36:25.716 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: debug 2023-11-17T05:36:14.553+0000 7f70ffa73700 4 rocksdb: Options.compression_opts.level: 32767 2023-11-17T05:36:25.716 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: debug 2023-11-17T05:36:14.553+0000 7f70ffa73700 4 rocksdb: Options.compression_opts.strategy: 0 2023-11-17T05:36:25.716 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: debug 2023-11-17T05:36:14.553+0000 7f70ffa73700 4 rocksdb: Options.compression_opts.max_dict_bytes: 0 2023-11-17T05:36:25.716 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: debug 2023-11-17T05:36:14.553+0000 7f70ffa73700 4 rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2023-11-17T05:36:25.716 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: debug 2023-11-17T05:36:14.553+0000 7f70ffa73700 4 rocksdb: Options.compression_opts.enabled: false 2023-11-17T05:36:25.716 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: debug 2023-11-17T05:36:14.553+0000 7f70ffa73700 4 rocksdb: Options.level0_file_num_compaction_trigger: 4 2023-11-17T05:36:25.716 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: debug 2023-11-17T05:36:14.553+0000 7f70ffa73700 4 rocksdb: Options.level0_slowdown_writes_trigger: 20 2023-11-17T05:36:25.716 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: debug 2023-11-17T05:36:14.553+0000 7f70ffa73700 4 rocksdb: Options.level0_stop_writes_trigger: 36 2023-11-17T05:36:25.716 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: debug 2023-11-17T05:36:14.553+0000 7f70ffa73700 4 rocksdb: Options.target_file_size_base: 67108864 2023-11-17T05:36:25.716 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: debug 2023-11-17T05:36:14.553+0000 7f70ffa73700 4 rocksdb: Options.target_file_size_multiplier: 1 2023-11-17T05:36:25.716 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: debug 2023-11-17T05:36:14.553+0000 7f70ffa73700 4 rocksdb: Options.max_bytes_for_level_base: 268435456 2023-11-17T05:36:25.716 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: debug 2023-11-17T05:36:14.553+0000 7f70ffa73700 4 rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2023-11-17T05:36:25.717 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: debug 2023-11-17T05:36:14.553+0000 7f70ffa73700 4 rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2023-11-17T05:36:25.717 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: debug 2023-11-17T05:36:14.553+0000 7f70ffa73700 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2023-11-17T05:36:25.717 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: debug 2023-11-17T05:36:14.553+0000 7f70ffa73700 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2023-11-17T05:36:25.717 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: debug 2023-11-17T05:36:14.553+0000 7f70ffa73700 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2023-11-17T05:36:25.717 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: debug 2023-11-17T05:36:14.553+0000 7f70ffa73700 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2023-11-17T05:36:25.717 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: debug 2023-11-17T05:36:14.553+0000 7f70ffa73700 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2023-11-17T05:36:25.717 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: debug 2023-11-17T05:36:14.553+0000 7f70ffa73700 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2023-11-17T05:36:25.717 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: debug 2023-11-17T05:36:14.553+0000 7f70ffa73700 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2023-11-17T05:36:25.717 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: debug 2023-11-17T05:36:14.553+0000 7f70ffa73700 4 rocksdb: Options.max_sequential_skip_in_iterations: 8 2023-11-17T05:36:25.717 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: debug 2023-11-17T05:36:14.553+0000 7f70ffa73700 4 rocksdb: Options.max_compaction_bytes: 1677721600 2023-11-17T05:36:25.824 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: debug 2023-11-17T05:36:14.553+0000 7f70ffa73700 4 rocksdb: Options.arena_block_size: 4194304 2023-11-17T05:36:25.824 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: debug 2023-11-17T05:36:14.553+0000 7f70ffa73700 4 rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2023-11-17T05:36:25.824 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: debug 2023-11-17T05:36:14.553+0000 7f70ffa73700 4 rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2023-11-17T05:36:25.824 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: debug 2023-11-17T05:36:14.553+0000 7f70ffa73700 4 rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2023-11-17T05:36:25.825 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: debug 2023-11-17T05:36:14.553+0000 7f70ffa73700 4 rocksdb: Options.disable_auto_compactions: 0 2023-11-17T05:36:25.825 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: debug 2023-11-17T05:36:14.553+0000 7f70ffa73700 4 rocksdb: Options.compaction_style: kCompactionStyleLevel 2023-11-17T05:36:25.825 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: debug 2023-11-17T05:36:14.553+0000 7f70ffa73700 4 rocksdb: Options.compaction_pri: kMinOverlappingRatio 2023-11-17T05:36:25.825 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: debug 2023-11-17T05:36:14.553+0000 7f70ffa73700 4 rocksdb: Options.compaction_options_universal.size_ratio: 1 2023-11-17T05:36:25.825 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: debug 2023-11-17T05:36:14.553+0000 7f70ffa73700 4 rocksdb: Options.compaction_options_universal.min_merge_width: 2 2023-11-17T05:36:25.825 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: debug 2023-11-17T05:36:14.553+0000 7f70ffa73700 4 rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2023-11-17T05:36:25.825 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: debug 2023-11-17T05:36:14.553+0000 7f70ffa73700 4 rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2023-11-17T05:36:25.825 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: debug 2023-11-17T05:36:14.553+0000 7f70ffa73700 4 rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2023-11-17T05:36:25.825 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: debug 2023-11-17T05:36:14.553+0000 7f70ffa73700 4 rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2023-11-17T05:36:25.825 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: debug 2023-11-17T05:36:14.553+0000 7f70ffa73700 4 rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2023-11-17T05:36:25.825 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: debug 2023-11-17T05:36:14.553+0000 7f70ffa73700 4 rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2023-11-17T05:36:25.825 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: debug 2023-11-17T05:36:14.553+0000 7f70ffa73700 4 rocksdb: Options.table_properties_collectors: 2023-11-17T05:36:25.825 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: debug 2023-11-17T05:36:14.553+0000 7f70ffa73700 4 rocksdb: Options.inplace_update_support: 0 2023-11-17T05:36:25.826 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: debug 2023-11-17T05:36:14.553+0000 7f70ffa73700 4 rocksdb: Options.inplace_update_num_locks: 10000 2023-11-17T05:36:25.826 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: debug 2023-11-17T05:36:14.553+0000 7f70ffa73700 4 rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2023-11-17T05:36:25.826 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: debug 2023-11-17T05:36:14.553+0000 7f70ffa73700 4 rocksdb: Options.memtable_whole_key_filtering: 0 2023-11-17T05:36:25.826 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: debug 2023-11-17T05:36:14.553+0000 7f70ffa73700 4 rocksdb: Options.memtable_huge_page_size: 0 2023-11-17T05:36:25.826 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: debug 2023-11-17T05:36:14.553+0000 7f70ffa73700 4 rocksdb: Options.bloom_locality: 0 2023-11-17T05:36:25.826 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: debug 2023-11-17T05:36:14.553+0000 7f70ffa73700 4 rocksdb: Options.max_successive_merges: 0 2023-11-17T05:36:25.826 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: debug 2023-11-17T05:36:14.553+0000 7f70ffa73700 4 rocksdb: Options.optimize_filters_for_hits: 0 2023-11-17T05:36:25.826 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: debug 2023-11-17T05:36:14.553+0000 7f70ffa73700 4 rocksdb: Options.paranoid_file_checks: 0 2023-11-17T05:36:25.826 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: debug 2023-11-17T05:36:14.553+0000 7f70ffa73700 4 rocksdb: Options.force_consistency_checks: 0 2023-11-17T05:36:25.826 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: debug 2023-11-17T05:36:14.553+0000 7f70ffa73700 4 rocksdb: Options.report_bg_io_stats: 0 2023-11-17T05:36:25.826 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: debug 2023-11-17T05:36:14.553+0000 7f70ffa73700 4 rocksdb: Options.ttl: 2592000 2023-11-17T05:36:25.826 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: debug 2023-11-17T05:36:14.553+0000 7f70ffa73700 4 rocksdb: Options.periodic_compaction_seconds: 0 2023-11-17T05:36:25.827 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: debug 2023-11-17T05:36:14.554+0000 7f70ffa73700 4 rocksdb: [version_set.cc:4568] Recovered from manifest file:/var/lib/ceph/mon/ceph-smithi066/store.db/MANIFEST-000001 succeeded,manifest_file_number is 1, next_file_number is 3, last_sequence is 0, log_number is 0,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2023-11-17T05:36:25.827 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: 2023-11-17T05:36:25.827 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: debug 2023-11-17T05:36:14.554+0000 7f70ffa73700 4 rocksdb: [version_set.cc:4577] Column family [default] (ID 0), log number is 0 2023-11-17T05:36:25.827 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: 2023-11-17T05:36:25.827 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: debug 2023-11-17T05:36:14.554+0000 7f70ffa73700 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1700199374554640, "job": 1, "event": "recovery_started", "log_files": [3]} 2023-11-17T05:36:25.827 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: debug 2023-11-17T05:36:14.554+0000 7f70ffa73700 4 rocksdb: [db_impl/db_impl_open.cc:760] Recovering log #3 mode 2 2023-11-17T05:36:25.827 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: debug 2023-11-17T05:36:14.554+0000 7f70ffa73700 3 rocksdb: [le/block_based/filter_policy.cc:584] Using legacy Bloom filter with high (20) bits/key. Dramatic filter space and/or accuracy improvement is available with format_version>=5. 2023-11-17T05:36:25.827 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: debug 2023-11-17T05:36:14.554+0000 7f70ffa73700 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1700199374555122, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 4, "file_size": 1437, "table_properties": {"data_size": 523, "index_size": 40, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 0, "index_value_is_delta_encoded": 0, "filter_size": 69, "raw_key_size": 115, "raw_average_key_size": 23, "raw_value_size": 401, "raw_average_value_size": 80, "num_data_blocks": 1, "num_entries": 5, "num_deletions": 0, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "rocksdb.BuiltinBloomFilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; ", "creation_time": 1700199374, "oldest_key_time": 3, "file_creation_time": 0}} 2023-11-17T05:36:25.827 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: debug 2023-11-17T05:36:14.554+0000 7f70ffa73700 4 rocksdb: [version_set.cc:3826] Creating manifest 5 2023-11-17T05:36:25.827 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: 2023-11-17T05:36:25.827 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: debug 2023-11-17T05:36:14.555+0000 7f70ffa73700 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1700199374555521, "job": 1, "event": "recovery_finished"} 2023-11-17T05:36:25.827 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: debug 2023-11-17T05:36:14.555+0000 7f70ffa73700 4 rocksdb: DB pointer 0x5631a6be1800 2023-11-17T05:36:25.827 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: debug 2023-11-17T05:36:14.555+0000 7f70e688b700 4 rocksdb: [db_impl/db_impl.cc:850] ------- DUMPING STATS ------- 2023-11-17T05:36:25.828 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: debug 2023-11-17T05:36:14.555+0000 7f70e688b700 4 rocksdb: [db_impl/db_impl.cc:851] 2023-11-17T05:36:25.828 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: ** DB Stats ** 2023-11-17T05:36:25.828 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: Uptime(secs): 0.0 total, 0.0 interval 2023-11-17T05:36:25.828 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2023-11-17T05:36:25.828 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2023-11-17T05:36:25.932 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2023-11-17T05:36:25.933 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2023-11-17T05:36:25.933 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2023-11-17T05:36:25.933 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2023-11-17T05:36:25.933 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: 2023-11-17T05:36:25.933 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: ** Compaction Stats [default] ** 2023-11-17T05:36:25.933 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-11-17T05:36:25.933 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-11-17T05:36:25.933 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: L0 1/0 1.40 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 3.1 0.00 0.00 1 0.000 0 0 2023-11-17T05:36:25.933 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: Sum 1/0 1.40 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 3.1 0.00 0.00 1 0.000 0 0 2023-11-17T05:36:25.933 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 3.1 0.00 0.00 1 0.000 0 0 2023-11-17T05:36:25.933 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: 2023-11-17T05:36:25.933 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: ** Compaction Stats [default] ** 2023-11-17T05:36:25.934 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-11-17T05:36:25.934 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-11-17T05:36:25.934 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 3.1 0.00 0.00 1 0.000 0 0 2023-11-17T05:36:25.934 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: Uptime(secs): 0.0 total, 0.0 interval 2023-11-17T05:36:25.934 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: Flush(GB): cumulative 0.000, interval 0.000 2023-11-17T05:36:25.934 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: AddFile(GB): cumulative 0.000, interval 0.000 2023-11-17T05:36:25.934 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: AddFile(Total Files): cumulative 0, interval 0 2023-11-17T05:36:25.934 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: AddFile(L0 Files): cumulative 0, interval 0 2023-11-17T05:36:25.934 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: AddFile(Keys): cumulative 0, interval 0 2023-11-17T05:36:25.934 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: Cumulative compaction: 0.00 GB write, 0.52 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-11-17T05:36:25.934 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: Interval compaction: 0.00 GB write, 0.52 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-11-17T05:36:25.934 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-11-17T05:36:25.934 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: 2023-11-17T05:36:25.935 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: ** File Read Latency Histogram By Level [default] ** 2023-11-17T05:36:25.935 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: 2023-11-17T05:36:25.935 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: ** Compaction Stats [default] ** 2023-11-17T05:36:25.935 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-11-17T05:36:25.935 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-11-17T05:36:25.935 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: L0 1/0 1.40 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 3.1 0.00 0.00 1 0.000 0 0 2023-11-17T05:36:25.935 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: Sum 1/0 1.40 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 3.1 0.00 0.00 1 0.000 0 0 2023-11-17T05:36:25.935 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2023-11-17T05:36:25.935 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: 2023-11-17T05:36:25.935 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: ** Compaction Stats [default] ** 2023-11-17T05:36:25.935 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-11-17T05:36:25.935 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-11-17T05:36:25.935 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 3.1 0.00 0.00 1 0.000 0 0 2023-11-17T05:36:25.936 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: Uptime(secs): 0.0 total, 0.0 interval 2023-11-17T05:36:25.936 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: Flush(GB): cumulative 0.000, interval 0.000 2023-11-17T05:36:25.936 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: AddFile(GB): cumulative 0.000, interval 0.000 2023-11-17T05:36:25.936 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: AddFile(Total Files): cumulative 0, interval 0 2023-11-17T05:36:26.084 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: AddFile(L0 Files): cumulative 0, interval 0 2023-11-17T05:36:26.084 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: AddFile(Keys): cumulative 0, interval 0 2023-11-17T05:36:26.084 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: Cumulative compaction: 0.00 GB write, 0.51 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-11-17T05:36:26.084 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-11-17T05:36:26.085 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-11-17T05:36:26.085 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: 2023-11-17T05:36:26.085 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: ** File Read Latency Histogram By Level [default] ** 2023-11-17T05:36:26.085 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: 2023-11-17T05:36:26.085 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: debug 2023-11-17T05:36:14.556+0000 7f70ffa73700 0 mon.smithi066 does not exist in monmap, will attempt to join an existing cluster 2023-11-17T05:36:26.085 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: debug 2023-11-17T05:36:14.556+0000 7f70ffa73700 0 using public_addr v2:172.21.15.66:0/0 -> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] 2023-11-17T05:36:26.085 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: debug 2023-11-17T05:36:14.556+0000 7f70ffa73700 0 starting mon.smithi066 rank -1 at public addrs [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] at bind addrs [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] mon_data /var/lib/ceph/mon/ceph-smithi066 fsid ed6470e2-8508-11ee-8dbe-212e2dc638e7 2023-11-17T05:36:26.085 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: debug 2023-11-17T05:36:14.557+0000 7f70ffa73700 1 mon.smithi066@-1(???) e0 preinit fsid ed6470e2-8508-11ee-8dbe-212e2dc638e7 2023-11-17T05:36:26.085 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: debug 2023-11-17T05:36:14.560+0000 7f70e9090700 1 mon.smithi066@-1(synchronizing) e1 sync_obtain_latest_monmap 2023-11-17T05:36:26.085 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: debug 2023-11-17T05:36:14.560+0000 7f70e9090700 1 mon.smithi066@-1(synchronizing) e1 sync_obtain_latest_monmap obtained monmap e1 2023-11-17T05:36:26.085 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 systemd[1]: Started Ceph mon.smithi066 for ed6470e2-8508-11ee-8dbe-212e2dc638e7. 2023-11-17T05:36:26.085 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: debug 2023-11-17T05:36:14.730+0000 7f70e9090700 4 rocksdb: [db_impl/db_impl_write.cc:1668] [default] New memtable created with log file: #9. Immutable memtables: 0. 2023-11-17T05:36:26.085 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: 2023-11-17T05:36:26.086 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: debug 2023-11-17T05:36:14.730+0000 7f70ee89b700 4 rocksdb: (Original Log Time 2023/11/17-05:36:14.730664) [db_impl/db_impl_compaction_flush.cc:2198] Calling FlushMemTableToOutputFile with column family [default], flush slots available 1, compaction slots available 1, flush slots scheduled 1, compaction slots scheduled 0 2023-11-17T05:36:26.086 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: debug 2023-11-17T05:36:14.730+0000 7f70ee89b700 4 rocksdb: [flush_job.cc:321] [default] [JOB 3] Flushing memtable with next log file: 9 2023-11-17T05:36:26.086 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: 2023-11-17T05:36:26.086 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: debug 2023-11-17T05:36:14.730+0000 7f70ee89b700 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1700199374730736, "job": 3, "event": "flush_started", "num_memtables": 1, "num_entries": 2047, "num_deletes": 0, "total_data_size": 32648894, "memory_usage": 32754648, "flush_reason": "Write Buffer Full"} 2023-11-17T05:36:26.086 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: debug 2023-11-17T05:36:14.730+0000 7f70ee89b700 4 rocksdb: [flush_job.cc:350] [default] [JOB 3] Level-0 flush table #10: started 2023-11-17T05:36:26.086 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: debug 2023-11-17T05:36:14.781+0000 7f70ee89b700 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1700199374782278, "cf_name": "default", "job": 3, "event": "table_file_creation", "file_number": 10, "file_size": 32684169, "table_properties": {"data_size": 32648256, "index_size": 29876, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 0, "index_value_is_delta_encoded": 0, "filter_size": 5189, "raw_key_size": 46772, "raw_average_key_size": 22, "raw_value_size": 32595489, "raw_average_value_size": 15923, "num_data_blocks": 890, "num_entries": 2047, "num_deletions": 0, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "rocksdb.BuiltinBloomFilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; ", "creation_time": 1700199374, "oldest_key_time": 1700199374, "file_creation_time": 1700199374}} 2023-11-17T05:36:26.086 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: debug 2023-11-17T05:36:14.781+0000 7f70ee89b700 4 rocksdb: [flush_job.cc:401] [default] [JOB 3] Level-0 flush table #10: 32684169 bytes OK 2023-11-17T05:36:26.086 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: debug 2023-11-17T05:36:14.782+0000 7f70ee89b700 4 rocksdb: (Original Log Time 2023/11/17-05:36:14.782332) [memtable_list.cc:447] [default] Level-0 commit table #10 started 2023-11-17T05:36:26.086 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: debug 2023-11-17T05:36:14.782+0000 7f70ee89b700 4 rocksdb: (Original Log Time 2023/11/17-05:36:14.782916) [memtable_list.cc:503] [default] Level-0 commit table #10: memtable #1 done 2023-11-17T05:36:26.086 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: debug 2023-11-17T05:36:14.782+0000 7f70ee89b700 4 rocksdb: (Original Log Time 2023/11/17-05:36:14.782928) EVENT_LOG_v1 {"time_micros": 1700199374782923, "job": 3, "event": "flush_finished", "output_compression": "NoCompression", "lsm_state": [2, 0, 0, 0, 0, 0, 0], "immutable_memtables": 0} 2023-11-17T05:36:26.086 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: debug 2023-11-17T05:36:14.782+0000 7f70ee89b700 4 rocksdb: (Original Log Time 2023/11/17-05:36:14.782940) [db_impl/db_impl_compaction_flush.cc:205] [default] Level summary: files[2 0 0 0 0 0 0] max score 0.50 2023-11-17T05:36:26.087 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: 2023-11-17T05:36:26.087 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: debug 2023-11-17T05:36:14.782+0000 7f70ee89b700 4 rocksdb: [db_impl/db_impl_files.cc:353] [JOB 3] Try to delete WAL files size 32634973, prev total WAL file size 39883306, number of live WAL files 2. 2023-11-17T05:36:26.087 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: 2023-11-17T05:36:26.087 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: debug 2023-11-17T05:36:14.782+0000 7f70e9090700 0 mon.smithi066@-1(synchronizing).mds e1 new map 2023-11-17T05:36:26.087 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: debug 2023-11-17T05:36:14.782+0000 7f70e9090700 0 mon.smithi066@-1(synchronizing).mds e1 print_map 2023-11-17T05:36:26.087 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: e1 2023-11-17T05:36:26.087 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: enable_multiple, ever_enabled_multiple: 1,1 2023-11-17T05:36:26.087 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2023-11-17T05:36:26.087 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: legacy client fscid: -1 2023-11-17T05:36:26.113 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: 2023-11-17T05:36:26.113 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: No filesystems configured 2023-11-17T05:36:26.113 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: 2023-11-17T05:36:26.113 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: debug 2023-11-17T05:36:14.782+0000 7f70e9090700 0 mon.smithi066@-1(synchronizing).osd e5 crush map has features 3314932999778484224, adjusting msgr requires 2023-11-17T05:36:26.113 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: debug 2023-11-17T05:36:14.782+0000 7f70e9090700 0 mon.smithi066@-1(synchronizing).osd e5 crush map has features 288514050185494528, adjusting msgr requires 2023-11-17T05:36:26.113 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: debug 2023-11-17T05:36:14.782+0000 7f70e9090700 0 mon.smithi066@-1(synchronizing).osd e5 crush map has features 288514050185494528, adjusting msgr requires 2023-11-17T05:36:26.113 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: debug 2023-11-17T05:36:14.782+0000 7f70e9090700 0 mon.smithi066@-1(synchronizing).osd e5 crush map has features 288514050185494528, adjusting msgr requires 2023-11-17T05:36:26.113 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: cluster 2023-11-17T05:36:13.564863+0000 mgr.smithi040.ogcprl (mgr.14164) 466 : cluster [DBG] pgmap v446: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:36:26.113 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: debug 2023-11-17T05:36:14.783+0000 7f70e9090700 1 mon.smithi066@-1(synchronizing).paxosservice(auth 1..9) refresh upgraded, format 0 -> 3 2023-11-17T05:36:26.113 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: debug 2023-11-17T05:36:14.784+0000 7f70e9090700 20 expand_channel_meta expand map: {default=false} 2023-11-17T05:36:26.113 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: debug 2023-11-17T05:36:14.784+0000 7f70e9090700 20 expand_channel_meta from 'false' to 'false' 2023-11-17T05:36:26.113 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: debug 2023-11-17T05:36:14.784+0000 7f70e9090700 20 expand_channel_meta expanded map: {default=false} 2023-11-17T05:36:26.114 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: debug 2023-11-17T05:36:14.784+0000 7f70e9090700 20 expand_channel_meta expand map: {default=info} 2023-11-17T05:36:26.114 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: debug 2023-11-17T05:36:14.784+0000 7f70e9090700 20 expand_channel_meta from 'info' to 'info' 2023-11-17T05:36:26.114 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: debug 2023-11-17T05:36:14.784+0000 7f70e9090700 20 expand_channel_meta expanded map: {default=info} 2023-11-17T05:36:26.114 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: debug 2023-11-17T05:36:14.784+0000 7f70e9090700 20 expand_channel_meta expand map: {default=daemon} 2023-11-17T05:36:26.114 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: debug 2023-11-17T05:36:14.784+0000 7f70e9090700 20 expand_channel_meta from 'daemon' to 'daemon' 2023-11-17T05:36:26.114 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: debug 2023-11-17T05:36:14.784+0000 7f70e9090700 20 expand_channel_meta expanded map: {default=daemon} 2023-11-17T05:36:26.114 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: debug 2023-11-17T05:36:14.784+0000 7f70e9090700 20 expand_channel_meta expand map: {default=debug} 2023-11-17T05:36:26.114 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: debug 2023-11-17T05:36:14.784+0000 7f70e9090700 20 expand_channel_meta from 'debug' to 'debug' 2023-11-17T05:36:26.114 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: debug 2023-11-17T05:36:14.784+0000 7f70e9090700 20 expand_channel_meta expanded map: {default=debug} 2023-11-17T05:36:26.114 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:14 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: debug 2023-11-17T05:36:14.784+0000 7f70e9090700 10 mon.smithi066@-1(synchronizing) e1 handle_conf_change mon_allow_pool_delete 2023-11-17T05:36:26.114 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:19 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: cephadm 2023-11-17T05:36:14.614623+0000 mgr.smithi040.ogcprl (mgr.14164) 467 : cephadm [INF] Deploying daemon node-exporter.smithi066 on smithi066 2023-11-17T05:36:26.114 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:19 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: audit 2023-11-17T05:36:14.790557+0000 mon.smithi040 (mon.0) 359 : audit [DBG] from='mgr.14164 172.21.15.40:0/2203032451' entity='mgr.smithi040.ogcprl' cmd=[{"prefix": "mon metadata", "id": "smithi040"}]: dispatch 2023-11-17T05:36:26.114 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:19 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: cluster 2023-11-17T05:36:14.790833+0000 mon.smithi040 (mon.0) 360 : cluster [INF] mon.smithi040 calling monitor election 2023-11-17T05:36:26.115 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:19 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: audit 2023-11-17T05:36:14.803564+0000 mon.smithi040 (mon.0) 361 : audit [DBG] from='mgr.14164 172.21.15.40:0/2203032451' entity='mgr.smithi040.ogcprl' cmd=[{"prefix": "mon metadata", "id": "smithi066"}]: dispatch 2023-11-17T05:36:26.115 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:19 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: cluster 2023-11-17T05:36:15.565194+0000 mgr.smithi040.ogcprl (mgr.14164) 468 : cluster [DBG] pgmap v447: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:36:26.115 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:19 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: audit 2023-11-17T05:36:15.786951+0000 mon.smithi040 (mon.0) 362 : audit [DBG] from='mgr.14164 172.21.15.40:0/2203032451' entity='mgr.smithi040.ogcprl' cmd=[{"prefix": "mon metadata", "id": "smithi066"}]: dispatch 2023-11-17T05:36:26.115 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:19 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: audit 2023-11-17T05:36:16.787339+0000 mon.smithi040 (mon.0) 363 : audit [DBG] from='mgr.14164 172.21.15.40:0/2203032451' entity='mgr.smithi040.ogcprl' cmd=[{"prefix": "mon metadata", "id": "smithi066"}]: dispatch 2023-11-17T05:36:26.115 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:19 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: cluster 2023-11-17T05:36:16.788315+0000 mon.smithi066 (mon.1) 1 : cluster [INF] mon.smithi066 calling monitor election 2023-11-17T05:36:26.115 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:19 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: cluster 2023-11-17T05:36:17.565574+0000 mgr.smithi040.ogcprl (mgr.14164) 469 : cluster [DBG] pgmap v448: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:36:26.115 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:19 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: audit 2023-11-17T05:36:17.626870+0000 mon.smithi040 (mon.0) 364 : audit [DBG] from='mgr.? 172.21.15.66:0/3940103742' entity='mgr.smithi066.aazujw' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi066.aazujw/crt"}]: dispatch 2023-11-17T05:36:26.115 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:19 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: audit 2023-11-17T05:36:17.787413+0000 mon.smithi040 (mon.0) 365 : audit [DBG] from='mgr.14164 172.21.15.40:0/2203032451' entity='mgr.smithi040.ogcprl' cmd=[{"prefix": "mon metadata", "id": "smithi066"}]: dispatch 2023-11-17T05:36:26.115 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:19 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: audit 2023-11-17T05:36:18.787462+0000 mon.smithi040 (mon.0) 366 : audit [DBG] from='mgr.14164 172.21.15.40:0/2203032451' entity='mgr.smithi040.ogcprl' cmd=[{"prefix": "mon metadata", "id": "smithi066"}]: dispatch 2023-11-17T05:36:26.115 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:19 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: cluster 2023-11-17T05:36:19.565969+0000 mgr.smithi040.ogcprl (mgr.14164) 470 : cluster [DBG] pgmap v449: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:36:26.115 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:19 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: audit 2023-11-17T05:36:19.787586+0000 mon.smithi040 (mon.0) 367 : audit [DBG] from='mgr.14164 172.21.15.40:0/2203032451' entity='mgr.smithi040.ogcprl' cmd=[{"prefix": "mon metadata", "id": "smithi066"}]: dispatch 2023-11-17T05:36:26.115 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:19 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: cluster 2023-11-17T05:36:19.817804+0000 mon.smithi040 (mon.0) 368 : cluster [INF] mon.smithi040 is new leader, mons smithi040,smithi066 in quorum (ranks 0,1) 2023-11-17T05:36:26.115 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:19 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: cluster 2023-11-17T05:36:19.822279+0000 mon.smithi040 (mon.0) 369 : cluster [DBG] monmap e2: 2 mons at {smithi040=[v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0],smithi066=[v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0]} 2023-11-17T05:36:26.116 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:19 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: cluster 2023-11-17T05:36:19.822328+0000 mon.smithi040 (mon.0) 370 : cluster [DBG] fsmap 2023-11-17T05:36:26.116 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:19 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: cluster 2023-11-17T05:36:19.822358+0000 mon.smithi040 (mon.0) 371 : cluster [DBG] osdmap e5: 0 total, 0 up, 0 in 2023-11-17T05:36:26.116 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:19 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: cluster 2023-11-17T05:36:19.822629+0000 mon.smithi040 (mon.0) 372 : cluster [DBG] mgrmap e17: smithi040.ogcprl(active, since 15m) 2023-11-17T05:36:26.116 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:19 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: cluster 2023-11-17T05:36:19.823728+0000 mon.smithi040 (mon.0) 373 : cluster [DBG] Standby manager daemon smithi066.aazujw started 2023-11-17T05:36:26.116 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:19 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: audit 2023-11-17T05:36:19.824763+0000 mon.smithi040 (mon.0) 374 : audit [DBG] from='mgr.? 172.21.15.66:0/3940103742' entity='mgr.smithi066.aazujw' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-11-17T05:36:26.116 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:19 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: cluster 2023-11-17T05:36:19.826546+0000 mon.smithi040 (mon.0) 375 : cluster [WRN] Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2023-11-17T05:36:26.116 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:19 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: cluster 2023-11-17T05:36:19.826575+0000 mon.smithi040 (mon.0) 376 : cluster [WRN] [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2023-11-17T05:36:26.116 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:19 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: cluster 2023-11-17T05:36:19.826583+0000 mon.smithi040 (mon.0) 377 : cluster [WRN] daemon node-exporter.smithi040 on smithi040 is in unknown state 2023-11-17T05:36:26.116 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:19 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: audit 2023-11-17T05:36:19.826854+0000 mon.smithi040 (mon.0) 378 : audit [DBG] from='mgr.? 172.21.15.66:0/3940103742' entity='mgr.smithi066.aazujw' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi066.aazujw/key"}]: dispatch 2023-11-17T05:36:26.169 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:19 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: audit 2023-11-17T05:36:19.827365+0000 mon.smithi040 (mon.0) 379 : audit [DBG] from='mgr.? 172.21.15.66:0/3940103742' entity='mgr.smithi066.aazujw' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-11-17T05:36:26.170 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:20 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: cluster 2023-11-17T05:36:19.885592+0000 mon.smithi040 (mon.0) 380 : cluster [DBG] mgrmap e18: smithi040.ogcprl(active, since 15m), standbys: smithi066.aazujw 2023-11-17T05:36:26.170 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:20 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: audit 2023-11-17T05:36:19.885784+0000 mon.smithi040 (mon.0) 381 : audit [DBG] from='mgr.14164 172.21.15.40:0/2203032451' entity='mgr.smithi040.ogcprl' cmd=[{"prefix": "mgr metadata", "who": "smithi066.aazujw", "id": "smithi066.aazujw"}]: dispatch 2023-11-17T05:36:26.170 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:20 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: audit 2023-11-17T05:36:20.787914+0000 mon.smithi040 (mon.0) 382 : audit [DBG] from='mgr.14164 172.21.15.40:0/2203032451' entity='mgr.smithi040.ogcprl' cmd=[{"prefix": "mon metadata", "id": "smithi066"}]: dispatch 2023-11-17T05:36:26.170 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:21 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: cluster 2023-11-17T05:36:21.566301+0000 mgr.smithi040.ogcprl (mgr.14164) 471 : cluster [DBG] pgmap v450: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:36:26.170 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:23 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: cluster 2023-11-17T05:36:23.566632+0000 mgr.smithi040.ogcprl (mgr.14164) 472 : cluster [DBG] pgmap v451: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:36:26.171 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:25 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: cluster 2023-11-17T05:36:25.566949+0000 mgr.smithi040.ogcprl (mgr.14164) 473 : cluster [DBG] pgmap v452: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:36:26.172 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:35:40 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:35:39.558417+0000 mgr.smithi040.ogcprl (mgr.14164) 445 : cluster [DBG] pgmap v429: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:36:26.172 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:35:41 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:35:41.558853+0000 mgr.smithi040.ogcprl (mgr.14164) 446 : cluster [DBG] pgmap v430: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:36:26.172 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:35:43 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:35:43.559228+0000 mgr.smithi040.ogcprl (mgr.14164) 447 : cluster [DBG] pgmap v431: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:36:26.172 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:35:45 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:35:45.559597+0000 mgr.smithi040.ogcprl (mgr.14164) 448 : cluster [DBG] pgmap v432: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:36:26.172 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:35:47 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:35:47.559908+0000 mgr.smithi040.ogcprl (mgr.14164) 449 : cluster [DBG] pgmap v433: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:36:26.172 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:35:50 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:35:49.560269+0000 mgr.smithi040.ogcprl (mgr.14164) 450 : cluster [DBG] pgmap v434: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:36:26.172 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:35:51 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:35:51.560581+0000 mgr.smithi040.ogcprl (mgr.14164) 451 : cluster [DBG] pgmap v435: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:36:26.172 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:35:53 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:35:53.560907+0000 mgr.smithi040.ogcprl (mgr.14164) 452 : cluster [DBG] pgmap v436: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:36:26.172 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:35:55 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:35:55.561275+0000 mgr.smithi040.ogcprl (mgr.14164) 453 : cluster [DBG] pgmap v437: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:36:26.172 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:35:57 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:35:57.561706+0000 mgr.smithi040.ogcprl (mgr.14164) 454 : cluster [DBG] pgmap v438: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:36:26.172 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:36:00 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:35:59.562060+0000 mgr.smithi040.ogcprl (mgr.14164) 455 : cluster [DBG] pgmap v439: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:36:26.173 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:36:01 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:36:01.562452+0000 mgr.smithi040.ogcprl (mgr.14164) 456 : cluster [DBG] pgmap v440: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:36:26.173 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:36:03 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:36:03.562771+0000 mgr.smithi040.ogcprl (mgr.14164) 457 : cluster [DBG] pgmap v441: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:36:26.173 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:36:05 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:36:05.563131+0000 mgr.smithi040.ogcprl (mgr.14164) 458 : cluster [DBG] pgmap v442: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:36:26.173 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:36:07 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:36:07.563561+0000 mgr.smithi040.ogcprl (mgr.14164) 459 : cluster [DBG] pgmap v443: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:36:26.173 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:36:09 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:36:08.982093+0000 mon.smithi040 (mon.0) 344 : audit [INF] from='mgr.14164 172.21.15.40:0/2203032451' entity='mgr.smithi040.ogcprl' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi066", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2023-11-17T05:36:26.173 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:36:09 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:36:08.984738+0000 mon.smithi040 (mon.0) 345 : audit [INF] from='mgr.14164 172.21.15.40:0/2203032451' entity='mgr.smithi040.ogcprl' cmd='[{"prefix": "auth get-or-create", "entity": "client.crash.smithi066", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]': finished 2023-11-17T05:36:26.173 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:36:09 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:36:08.985626+0000 mon.smithi040 (mon.0) 346 : audit [DBG] from='mgr.14164 172.21.15.40:0/2203032451' entity='mgr.smithi040.ogcprl' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-11-17T05:36:26.173 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:36:10 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cephadm 2023-11-17T05:36:08.980718+0000 mgr.smithi040.ogcprl (mgr.14164) 460 : cephadm [ERR] cephadm exited with an error code: 1, stderr:Deploy daemon alertmanager.smithi066 ... 2023-11-17T05:36:26.173 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:36:10 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: Non-zero exit code 125 from /bin/podman run --rm --ipc=host --no-hosts --net=host --entrypoint stat --init -e CONTAINER_IMAGE=docker.io/prom/alertmanager:v0.20.0 -e NODE_NAME=smithi066 -e CEPH_USE_RANDOM_NONCE=1 docker.io/prom/alertmanager:v0.20.0 -c %u %g /etc/alertmanager 2023-11-17T05:36:26.173 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:36:10 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: stat: stderr Trying to pull docker.io/prom/alertmanager:v0.20.0... 2023-11-17T05:36:26.173 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:36:10 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: stat: stderr time="2023-11-17T05:31:48Z" level=warning msg="Failed, retrying in 1s ... (1/3). Error: initializing source docker://prom/alertmanager:v0.20.0: (Mirrors also failed: [docker-mirror.front.sepia.ceph.com:5000/prom/alertmanager:v0.20.0: pinging container registry docker-mirror.front.sepia.ceph.com:5000: Get \"https://docker-mirror.front.sepia.ceph.com:5000/v2/\": dial tcp 172.21.0.79:5000: connect: no route to host]): docker.io/prom/alertmanager:v0.20.0: reading manifest v0.20.0 in docker.io/prom/alertmanager: toomanyrequests: You have reached your pull rate limit. You may increase the limit by authenticating and upgrading: https://www.docker.com/increase-rate-limit" 2023-11-17T05:36:26.173 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:36:10 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: stat: stderr time="2023-11-17T05:32:25Z" level=warning msg="Failed, retrying in 1s ... (2/3). Error: initializing source docker://prom/alertmanager:v0.20.0: (Mirrors also failed: [docker-mirror.front.sepia.ceph.com:5000/prom/alertmanager:v0.20.0: pinging container registry docker-mirror.front.sepia.ceph.com:5000: Get \"https://docker-mirror.front.sepia.ceph.com:5000/v2/\": dial tcp 172.21.0.79:5000: connect: no route to host]): docker.io/prom/alertmanager:v0.20.0: reading manifest v0.20.0 in docker.io/prom/alertmanager: toomanyrequests: You have reached your pull rate limit. You may increase the limit by authenticating and upgrading: https://www.docker.com/increase-rate-limit" 2023-11-17T05:36:26.173 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:36:10 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: stat: stderr time="2023-11-17T05:33:02Z" level=warning msg="Failed, retrying in 1s ... (3/3). Error: initializing source docker://prom/alertmanager:v0.20.0: (Mirrors also failed: [docker-mirror.front.sepia.ceph.com:5000/prom/alertmanager:v0.20.0: pinging container registry docker-mirror.front.sepia.ceph.com:5000: Get \"https://docker-mirror.front.sepia.ceph.com:5000/v2/\": dial tcp 172.21.0.79:5000: connect: no route to host]): docker.io/prom/alertmanager:v0.20.0: reading manifest v0.20.0 in docker.io/prom/alertmanager: toomanyrequests: You have reached your pull rate limit. You may increase the limit by authenticating and upgrading: https://www.docker.com/increase-rate-limit" 2023-11-17T05:36:26.174 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:36:10 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: stat: stderr Error: initializing source docker://prom/alertmanager:v0.20.0: (Mirrors also failed: [docker-mirror.front.sepia.ceph.com:5000/prom/alertmanager:v0.20.0: pinging container registry docker-mirror.front.sepia.ceph.com:5000: Get "https://docker-mirror.front.sepia.ceph.com:5000/v2/": dial tcp 172.21.0.79:5000: connect: no route to host]): docker.io/prom/alertmanager:v0.20.0: reading manifest v0.20.0 in docker.io/prom/alertmanager: toomanyrequests: You have reached your pull rate limit. You may increase the limit by authenticating and upgrading: https://www.docker.com/increase-rate-limit 2023-11-17T05:36:26.174 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:36:10 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: Non-zero exit code 125 from /bin/podman run --rm --ipc=host --no-hosts --net=host --entrypoint stat --init -e CONTAINER_IMAGE=docker.io/prom/alertmanager:v0.20.0 -e NODE_NAME=smithi066 -e CEPH_USE_RANDOM_NONCE=1 docker.io/prom/alertmanager:v0.20.0 -c %u %g /etc/prometheus 2023-11-17T05:36:26.291 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:36:10 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: stat: stderr Trying to pull docker.io/prom/alertmanager:v0.20.0... 2023-11-17T05:36:26.291 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:36:10 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: stat: stderr time="2023-11-17T05:34:16Z" level=warning msg="Failed, retrying in 1s ... (1/3). Error: initializing source docker://prom/alertmanager:v0.20.0: (Mirrors also failed: [docker-mirror.front.sepia.ceph.com:5000/prom/alertmanager:v0.20.0: pinging container registry docker-mirror.front.sepia.ceph.com:5000: Get \"https://docker-mirror.front.sepia.ceph.com:5000/v2/\": dial tcp 172.21.0.79:5000: connect: no route to host]): docker.io/prom/alertmanager:v0.20.0: reading manifest v0.20.0 in docker.io/prom/alertmanager: toomanyrequests: You have reached your pull rate limit. You may increase the limit by authenticating and upgrading: https://www.docker.com/increase-rate-limit" 2023-11-17T05:36:26.291 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:36:10 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: stat: stderr time="2023-11-17T05:34:54Z" level=warning msg="Failed, retrying in 1s ... (2/3). Error: initializing source docker://prom/alertmanager:v0.20.0: (Mirrors also failed: [docker-mirror.front.sepia.ceph.com:5000/prom/alertmanager:v0.20.0: pinging container registry docker-mirror.front.sepia.ceph.com:5000: Get \"https://docker-mirror.front.sepia.ceph.com:5000/v2/\": dial tcp 172.21.0.79:5000: connect: no route to host]): docker.io/prom/alertmanager:v0.20.0: reading manifest v0.20.0 in docker.io/prom/alertmanager: toomanyrequests: You have reached your pull rate limit. You may increase the limit by authenticating and upgrading: https://www.docker.com/increase-rate-limit" 2023-11-17T05:36:26.291 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:36:10 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: stat: stderr time="2023-11-17T05:35:31Z" level=warning msg="Failed, retrying in 1s ... (3/3). Error: initializing source docker://prom/alertmanager:v0.20.0: (Mirrors also failed: [docker-mirror.front.sepia.ceph.com:5000/prom/alertmanager:v0.20.0: pinging container registry docker-mirror.front.sepia.ceph.com:5000: Get \"https://docker-mirror.front.sepia.ceph.com:5000/v2/\": dial tcp 172.21.0.79:5000: connect: no route to host]): docker.io/prom/alertmanager:v0.20.0: reading manifest v0.20.0 in docker.io/prom/alertmanager: toomanyrequests: You have reached your pull rate limit. You may increase the limit by authenticating and upgrading: https://www.docker.com/increase-rate-limit" 2023-11-17T05:36:26.292 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:36:10 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: stat: stderr Error: initializing source docker://prom/alertmanager:v0.20.0: (Mirrors also failed: [docker-mirror.front.sepia.ceph.com:5000/prom/alertmanager:v0.20.0: pinging container registry docker-mirror.front.sepia.ceph.com:5000: Get "https://docker-mirror.front.sepia.ceph.com:5000/v2/": dial tcp 172.21.0.79:5000: connect: no route to host]): docker.io/prom/alertmanager:v0.20.0: reading manifest v0.20.0 in docker.io/prom/alertmanager: toomanyrequests: You have reached your pull rate limit. You may increase the limit by authenticating and upgrading: https://www.docker.com/increase-rate-limit 2023-11-17T05:36:26.292 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:36:10 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: Traceback (most recent call last): 2023-11-17T05:36:26.292 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:36:10 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: File "/var/lib/ceph/ed6470e2-8508-11ee-8dbe-212e2dc638e7/cephadm.30cb78bdbbafb384af862e1c2292b944f15942b586128e91262b43e91e11ae90", line 8029, in 2023-11-17T05:36:26.292 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:36:10 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: main() 2023-11-17T05:36:26.292 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:36:10 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: File "/var/lib/ceph/ed6470e2-8508-11ee-8dbe-212e2dc638e7/cephadm.30cb78bdbbafb384af862e1c2292b944f15942b586128e91262b43e91e11ae90", line 8017, in main 2023-11-17T05:36:26.292 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:36:10 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: r = ctx.func(ctx) 2023-11-17T05:36:26.292 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:36:10 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: File "/var/lib/ceph/ed6470e2-8508-11ee-8dbe-212e2dc638e7/cephadm.30cb78bdbbafb384af862e1c2292b944f15942b586128e91262b43e91e11ae90", line 1760, in _default_image 2023-11-17T05:36:26.292 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:36:10 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: return func(ctx) 2023-11-17T05:36:26.292 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:36:10 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: File "/var/lib/ceph/ed6470e2-8508-11ee-8dbe-212e2dc638e7/cephadm.30cb78bdbbafb384af862e1c2292b944f15942b586128e91262b43e91e11ae90", line 4263, in command_deploy 2023-11-17T05:36:26.292 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:36:10 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: uid, gid = extract_uid_gid_monitoring(ctx, daemon_type) 2023-11-17T05:36:26.292 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:36:10 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: File "/var/lib/ceph/ed6470e2-8508-11ee-8dbe-212e2dc638e7/cephadm.30cb78bdbbafb384af862e1c2292b944f15942b586128e91262b43e91e11ae90", line 4191, in extract_uid_gid_monitoring 2023-11-17T05:36:26.292 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:36:10 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: uid, gid = extract_uid_gid(ctx, file_path=['/etc/alertmanager', '/etc/prometheus']) 2023-11-17T05:36:26.292 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:36:10 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: File "/var/lib/ceph/ed6470e2-8508-11ee-8dbe-212e2dc638e7/cephadm.30cb78bdbbafb384af862e1c2292b944f15942b586128e91262b43e91e11ae90", line 2531, in extract_uid_gid 2023-11-17T05:36:26.293 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:36:10 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: raise RuntimeError('uid/gid not found') 2023-11-17T05:36:26.293 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:36:10 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: RuntimeError: uid/gid not found 2023-11-17T05:36:26.293 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:36:10 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: Traceback (most recent call last): 2023-11-17T05:36:26.293 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:36:10 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1144, in _remote_connection 2023-11-17T05:36:26.293 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:36:10 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: yield (conn, connr) 2023-11-17T05:36:26.293 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:36:10 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: File "/usr/share/ceph/mgr/cephadm/serve.py", line 1059, in _run_cephadm 2023-11-17T05:36:26.293 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:36:10 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: code, '\n'.join(err))) 2023-11-17T05:36:26.293 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:36:10 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: orchestrator._interface.OrchestratorError: cephadm exited with an error code: 1, stderr:Deploy daemon alertmanager.smithi066 ... 2023-11-17T05:36:26.293 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:36:10 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: Non-zero exit code 125 from /bin/podman run --rm --ipc=host --no-hosts --net=host --entrypoint stat --init -e CONTAINER_IMAGE=docker.io/prom/alertmanager:v0.20.0 -e NODE_NAME=smithi066 -e CEPH_USE_RANDOM_NONCE=1 docker.io/prom/alertmanager:v0.20.0 -c %u %g /etc/alertmanager 2023-11-17T05:36:26.293 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:36:10 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: stat: stderr Trying to pull docker.io/prom/alertmanager:v0.20.0... 2023-11-17T05:36:26.293 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:36:10 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: stat: stderr time="2023-11-17T05:31:48Z" level=warning msg="Failed, retrying in 1s ... (1/3). Error: initializing source docker://prom/alertmanager:v0.20.0: (Mirrors also failed: [docker-mirror.front.sepia.ceph.com:5000/prom/alertmanager:v0.20.0: pinging container registry docker-mirror.front.sepia.ceph.com:5000: Get \"https://docker-mirror.front.sepia.ceph.com:5000/v2/\": dial tcp 172.21.0.79:5000: connect: no route to host]): docker.io/prom/alertmanager:v0.20.0: reading manifest v0.20.0 in docker.io/prom/alertmanager: toomanyrequests: You have reached your pull rate limit. You may increase the limit by authenticating and upgrading: https://www.docker.com/increase-rate-limit" 2023-11-17T05:36:26.293 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:36:10 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: stat: stderr time="2023-11-17T05:32:25Z" level=warning msg="Failed, retrying in 1s ... (2/3). Error: initializing source docker://prom/alertmanager:v0.20.0: (Mirrors also failed: [docker-mirror.front.sepia.ceph.com:5000/prom/alertmanager:v0.20.0: pinging container registry docker-mirror.front.sepia.ceph.com:5000: Get \"https://docker-mirror.front.sepia.ceph.com:5000/v2/\": dial tcp 172.21.0.79:5000: connect: no route to host]): docker.io/prom/alertmanager:v0.20.0: reading manifest v0.20.0 in docker.io/prom/alertmanager: toomanyrequests: You have reached your pull rate limit. You may increase the limit by authenticating and upgrading: https://www.docker.com/increase-rate-limit" 2023-11-17T05:36:26.293 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:36:10 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: stat: stderr time="2023-11-17T05:33:02Z" level=warning msg="Failed, retrying in 1s ... (3/3). Error: initializing source docker://prom/alertmanager:v0.20.0: (Mirrors also failed: [docker-mirror.front.sepia.ceph.com:5000/prom/alertmanager:v0.20.0: pinging container registry docker-mirror.front.sepia.ceph.com:5000: Get \"https://docker-mirror.front.sepia.ceph.com:5000/v2/\": dial tcp 172.21.0.79:5000: connect: no route to host]): docker.io/prom/alertmanager:v0.20.0: reading manifest v0.20.0 in docker.io/prom/alertmanager: toomanyrequests: You have reached your pull rate limit. You may increase the limit by authenticating and upgrading: https://www.docker.com/increase-rate-limit" 2023-11-17T05:36:26.294 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:36:10 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: stat: stderr Error: initializing source docker://prom/alertmanager:v0.20.0: (Mirrors also failed: [docker-mirror.front.sepia.ceph.com:5000/prom/alertmanager:v0.20.0: pinging container registry docker-mirror.front.sepia.ceph.com:5000: Get "https://docker-mirror.front.sepia.ceph.com:5000/v2/": dial tcp 172.21.0.79:5000: connect: no route to host]): docker.io/prom/alertmanager:v0.20.0: reading manifest v0.20.0 in docker.io/prom/alertmanager: toomanyrequests: You have reached your pull rate limit. You may increase the limit by authenticating and upgrading: https://www.docker.com/increase-rate-limit 2023-11-17T05:36:26.294 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:36:10 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: Non-zero exit code 125 from /bin/podman run --rm --ipc=host --no-hosts --net=host --entrypoint stat --init -e CONTAINER_IMAGE=docker.io/prom/alertmanager:v0.20.0 -e NODE_NAME=smithi066 -e CEPH_USE_RANDOM_NONCE=1 docker.io/prom/alertmanager:v0.20.0 -c %u %g /etc/prometheus 2023-11-17T05:36:26.294 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:36:10 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: stat: stderr Trying to pull docker.io/prom/alertmanager:v0.20.0... 2023-11-17T05:36:26.294 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:36:10 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: stat: stderr time="2023-11-17T05:34:16Z" level=warning msg="Failed, retrying in 1s ... (1/3). Error: initializing source docker://prom/alertmanager:v0.20.0: (Mirrors also failed: [docker-mirror.front.sepia.ceph.com:5000/prom/alertmanager:v0.20.0: pinging container registry docker-mirror.front.sepia.ceph.com:5000: Get \"https://docker-mirror.front.sepia.ceph.com:5000/v2/\": dial tcp 172.21.0.79:5000: connect: no route to host]): docker.io/prom/alertmanager:v0.20.0: reading manifest v0.20.0 in docker.io/prom/alertmanager: toomanyrequests: You have reached your pull rate limit. You may increase the limit by authenticating and upgrading: https://www.docker.com/increase-rate-limit" 2023-11-17T05:36:26.294 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:36:10 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: stat: stderr time="2023-11-17T05:34:54Z" level=warning msg="Failed, retrying in 1s ... (2/3). Error: initializing source docker://prom/alertmanager:v0.20.0: (Mirrors also failed: [docker-mirror.front.sepia.ceph.com:5000/prom/alertmanager:v0.20.0: pinging container registry docker-mirror.front.sepia.ceph.com:5000: Get \"https://docker-mirror.front.sepia.ceph.com:5000/v2/\": dial tcp 172.21.0.79:5000: connect: no route to host]): docker.io/prom/alertmanager:v0.20.0: reading manifest v0.20.0 in docker.io/prom/alertmanager: toomanyrequests: You have reached your pull rate limit. You may increase the limit by authenticating and upgrading: https://www.docker.com/increase-rate-limit" 2023-11-17T05:36:26.294 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:36:10 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: stat: stderr time="2023-11-17T05:35:31Z" level=warning msg="Failed, retrying in 1s ... (3/3). Error: initializing source docker://prom/alertmanager:v0.20.0: (Mirrors also failed: [docker-mirror.front.sepia.ceph.com:5000/prom/alertmanager:v0.20.0: pinging container registry docker-mirror.front.sepia.ceph.com:5000: Get \"https://docker-mirror.front.sepia.ceph.com:5000/v2/\": dial tcp 172.21.0.79:5000: connect: no route to host]): docker.io/prom/alertmanager:v0.20.0: reading manifest v0.20.0 in docker.io/prom/alertmanager: toomanyrequests: You have reached your pull rate limit. You may increase the limit by authenticating and upgrading: https://www.docker.com/increase-rate-limit" 2023-11-17T05:36:26.294 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:36:10 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: stat: stderr Error: initializing source docker://prom/alertmanager:v0.20.0: (Mirrors also failed: [docker-mirror.front.sepia.ceph.com:5000/prom/alertmanager:v0.20.0: pinging container registry docker-mirror.front.sepia.ceph.com:5000: Get "https://docker-mirror.front.sepia.ceph.com:5000/v2/": dial tcp 172.21.0.79:5000: connect: no route to host]): docker.io/prom/alertmanager:v0.20.0: reading manifest v0.20.0 in docker.io/prom/alertmanager: toomanyrequests: You have reached your pull rate limit. You may increase the limit by authenticating and upgrading: https://www.docker.com/increase-rate-limit 2023-11-17T05:36:26.441 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:36:10 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: Traceback (most recent call last): 2023-11-17T05:36:26.441 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:36:10 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: File "/var/lib/ceph/ed6470e2-8508-11ee-8dbe-212e2dc638e7/cephadm.30cb78bdbbafb384af862e1c2292b944f15942b586128e91262b43e91e11ae90", line 8029, in 2023-11-17T05:36:26.441 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:36:10 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: main() 2023-11-17T05:36:26.441 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:36:10 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: File "/var/lib/ceph/ed6470e2-8508-11ee-8dbe-212e2dc638e7/cephadm.30cb78bdbbafb384af862e1c2292b944f15942b586128e91262b43e91e11ae90", line 8017, in main 2023-11-17T05:36:26.441 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:36:10 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: r = ctx.func(ctx) 2023-11-17T05:36:26.441 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:36:10 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: File "/var/lib/ceph/ed6470e2-8508-11ee-8dbe-212e2dc638e7/cephadm.30cb78bdbbafb384af862e1c2292b944f15942b586128e91262b43e91e11ae90", line 1760, in _default_image 2023-11-17T05:36:26.441 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:36:10 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: return func(ctx) 2023-11-17T05:36:26.441 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:36:10 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: File "/var/lib/ceph/ed6470e2-8508-11ee-8dbe-212e2dc638e7/cephadm.30cb78bdbbafb384af862e1c2292b944f15942b586128e91262b43e91e11ae90", line 4263, in command_deploy 2023-11-17T05:36:26.441 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:36:10 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: uid, gid = extract_uid_gid_monitoring(ctx, daemon_type) 2023-11-17T05:36:26.441 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:36:10 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: File "/var/lib/ceph/ed6470e2-8508-11ee-8dbe-212e2dc638e7/cephadm.30cb78bdbbafb384af862e1c2292b944f15942b586128e91262b43e91e11ae90", line 4191, in extract_uid_gid_monitoring 2023-11-17T05:36:26.442 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:36:10 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: uid, gid = extract_uid_gid(ctx, file_path=['/etc/alertmanager', '/etc/prometheus']) 2023-11-17T05:36:26.442 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:36:10 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: File "/var/lib/ceph/ed6470e2-8508-11ee-8dbe-212e2dc638e7/cephadm.30cb78bdbbafb384af862e1c2292b944f15942b586128e91262b43e91e11ae90", line 2531, in extract_uid_gid 2023-11-17T05:36:26.442 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:36:10 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: raise RuntimeError('uid/gid not found') 2023-11-17T05:36:26.442 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:36:10 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: RuntimeError: uid/gid not found 2023-11-17T05:36:26.442 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:36:10 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cephadm 2023-11-17T05:36:08.986448+0000 mgr.smithi040.ogcprl (mgr.14164) 461 : cephadm [INF] Deploying daemon crash.smithi066 on smithi066 2023-11-17T05:36:26.442 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:36:10 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:36:09.564029+0000 mgr.smithi040.ogcprl (mgr.14164) 462 : cluster [DBG] pgmap v444: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:36:26.442 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:36:11 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:36:10.869303+0000 mon.smithi040 (mon.0) 347 : audit [INF] from='mgr.14164 172.21.15.40:0/2203032451' entity='mgr.smithi040.ogcprl' 2023-11-17T05:36:26.442 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:36:11 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:36:10.870906+0000 mon.smithi040 (mon.0) 348 : audit [INF] from='mgr.14164 172.21.15.40:0/2203032451' entity='mgr.smithi040.ogcprl' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi066.aazujw", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-11-17T05:36:26.442 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:36:11 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:36:10.873780+0000 mon.smithi040 (mon.0) 349 : audit [INF] from='mgr.14164 172.21.15.40:0/2203032451' entity='mgr.smithi040.ogcprl' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.smithi066.aazujw", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2023-11-17T05:36:26.442 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:36:11 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:36:10.874644+0000 mon.smithi040 (mon.0) 350 : audit [DBG] from='mgr.14164 172.21.15.40:0/2203032451' entity='mgr.smithi040.ogcprl' cmd=[{"prefix": "mgr services"}]: dispatch 2023-11-17T05:36:26.442 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:36:11 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:36:10.875697+0000 mon.smithi040 (mon.0) 351 : audit [DBG] from='mgr.14164 172.21.15.40:0/2203032451' entity='mgr.smithi040.ogcprl' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-11-17T05:36:26.442 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:36:11 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cephadm 2023-11-17T05:36:10.876470+0000 mgr.smithi040.ogcprl (mgr.14164) 463 : cephadm [INF] Deploying daemon mgr.smithi066.aazujw on smithi066 2023-11-17T05:36:26.443 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:36:11 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:36:11.324902+0000 mon.smithi040 (mon.0) 352 : audit [INF] from='mgr.14164 172.21.15.40:0/2203032451' entity='mgr.smithi040.ogcprl' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi040.ogcprl/mirror_snapshot_schedule"}]: dispatch 2023-11-17T05:36:26.443 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:36:11 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:36:11.325217+0000 mon.smithi040 (mon.0) 353 : audit [INF] from='mgr.14164 172.21.15.40:0/2203032451' entity='mgr.smithi040.ogcprl' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi040.ogcprl/trash_purge_schedule"}]: dispatch 2023-11-17T05:36:26.443 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:36:11 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:36:11.564456+0000 mgr.smithi040.ogcprl (mgr.14164) 464 : cluster [DBG] pgmap v445: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:36:26.443 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:36:13 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:36:12.373184+0000 mon.smithi040 (mon.0) 354 : audit [INF] from='mgr.14164 172.21.15.40:0/2203032451' entity='mgr.smithi040.ogcprl' 2023-11-17T05:36:26.443 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:36:13 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:36:12.374714+0000 mon.smithi040 (mon.0) 355 : audit [INF] from='mgr.14164 172.21.15.40:0/2203032451' entity='mgr.smithi040.ogcprl' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-11-17T05:36:26.443 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:36:13 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:36:12.375668+0000 mon.smithi040 (mon.0) 356 : audit [DBG] from='mgr.14164 172.21.15.40:0/2203032451' entity='mgr.smithi040.ogcprl' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-11-17T05:36:26.443 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:36:13 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cephadm 2023-11-17T05:36:12.376450+0000 mgr.smithi040.ogcprl (mgr.14164) 465 : cephadm [INF] Deploying daemon mon.smithi066 on smithi066 2023-11-17T05:36:26.443 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:36:14 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:36:13.564863+0000 mgr.smithi040.ogcprl (mgr.14164) 466 : cluster [DBG] pgmap v446: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:36:26.443 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:36:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cephadm 2023-11-17T05:36:14.614623+0000 mgr.smithi040.ogcprl (mgr.14164) 467 : cephadm [INF] Deploying daemon node-exporter.smithi066 on smithi066 2023-11-17T05:36:26.443 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:36:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:36:14.790557+0000 mon.smithi040 (mon.0) 359 : audit [DBG] from='mgr.14164 172.21.15.40:0/2203032451' entity='mgr.smithi040.ogcprl' cmd=[{"prefix": "mon metadata", "id": "smithi040"}]: dispatch 2023-11-17T05:36:26.443 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:36:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:36:14.790833+0000 mon.smithi040 (mon.0) 360 : cluster [INF] mon.smithi040 calling monitor election 2023-11-17T05:36:26.443 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:36:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:36:14.803564+0000 mon.smithi040 (mon.0) 361 : audit [DBG] from='mgr.14164 172.21.15.40:0/2203032451' entity='mgr.smithi040.ogcprl' cmd=[{"prefix": "mon metadata", "id": "smithi066"}]: dispatch 2023-11-17T05:36:26.443 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:36:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:36:15.565194+0000 mgr.smithi040.ogcprl (mgr.14164) 468 : cluster [DBG] pgmap v447: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:36:26.444 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:36:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:36:15.786951+0000 mon.smithi040 (mon.0) 362 : audit [DBG] from='mgr.14164 172.21.15.40:0/2203032451' entity='mgr.smithi040.ogcprl' cmd=[{"prefix": "mon metadata", "id": "smithi066"}]: dispatch 2023-11-17T05:36:26.444 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:36:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:36:16.787339+0000 mon.smithi040 (mon.0) 363 : audit [DBG] from='mgr.14164 172.21.15.40:0/2203032451' entity='mgr.smithi040.ogcprl' cmd=[{"prefix": "mon metadata", "id": "smithi066"}]: dispatch 2023-11-17T05:36:26.444 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:36:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:36:16.788315+0000 mon.smithi066 (mon.1) 1 : cluster [INF] mon.smithi066 calling monitor election 2023-11-17T05:36:26.444 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:36:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:36:17.565574+0000 mgr.smithi040.ogcprl (mgr.14164) 469 : cluster [DBG] pgmap v448: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:36:26.517 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:36:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:36:17.626870+0000 mon.smithi040 (mon.0) 364 : audit [DBG] from='mgr.? 172.21.15.66:0/3940103742' entity='mgr.smithi066.aazujw' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi066.aazujw/crt"}]: dispatch 2023-11-17T05:36:26.517 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:36:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:36:17.787413+0000 mon.smithi040 (mon.0) 365 : audit [DBG] from='mgr.14164 172.21.15.40:0/2203032451' entity='mgr.smithi040.ogcprl' cmd=[{"prefix": "mon metadata", "id": "smithi066"}]: dispatch 2023-11-17T05:36:26.517 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:36:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:36:18.787462+0000 mon.smithi040 (mon.0) 366 : audit [DBG] from='mgr.14164 172.21.15.40:0/2203032451' entity='mgr.smithi040.ogcprl' cmd=[{"prefix": "mon metadata", "id": "smithi066"}]: dispatch 2023-11-17T05:36:26.517 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:36:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:36:19.565969+0000 mgr.smithi040.ogcprl (mgr.14164) 470 : cluster [DBG] pgmap v449: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:36:26.517 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:36:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:36:19.787586+0000 mon.smithi040 (mon.0) 367 : audit [DBG] from='mgr.14164 172.21.15.40:0/2203032451' entity='mgr.smithi040.ogcprl' cmd=[{"prefix": "mon metadata", "id": "smithi066"}]: dispatch 2023-11-17T05:36:26.517 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:36:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:36:19.817804+0000 mon.smithi040 (mon.0) 368 : cluster [INF] mon.smithi040 is new leader, mons smithi040,smithi066 in quorum (ranks 0,1) 2023-11-17T05:36:26.517 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:36:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:36:19.822279+0000 mon.smithi040 (mon.0) 369 : cluster [DBG] monmap e2: 2 mons at {smithi040=[v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0],smithi066=[v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0]} 2023-11-17T05:36:26.517 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:36:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:36:19.822328+0000 mon.smithi040 (mon.0) 370 : cluster [DBG] fsmap 2023-11-17T05:36:26.517 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:36:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:36:19.822358+0000 mon.smithi040 (mon.0) 371 : cluster [DBG] osdmap e5: 0 total, 0 up, 0 in 2023-11-17T05:36:26.517 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:36:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:36:19.822629+0000 mon.smithi040 (mon.0) 372 : cluster [DBG] mgrmap e17: smithi040.ogcprl(active, since 15m) 2023-11-17T05:36:26.517 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:36:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:36:19.823728+0000 mon.smithi040 (mon.0) 373 : cluster [DBG] Standby manager daemon smithi066.aazujw started 2023-11-17T05:36:26.518 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:36:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:36:19.824763+0000 mon.smithi040 (mon.0) 374 : audit [DBG] from='mgr.? 172.21.15.66:0/3940103742' entity='mgr.smithi066.aazujw' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-11-17T05:36:26.518 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:36:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:36:19.826546+0000 mon.smithi040 (mon.0) 375 : cluster [WRN] Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2023-11-17T05:36:26.518 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:36:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:36:19.826575+0000 mon.smithi040 (mon.0) 376 : cluster [WRN] [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2023-11-17T05:36:26.518 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:36:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:36:19.826583+0000 mon.smithi040 (mon.0) 377 : cluster [WRN] daemon node-exporter.smithi040 on smithi040 is in unknown state 2023-11-17T05:36:26.518 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:36:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:36:19.826854+0000 mon.smithi040 (mon.0) 378 : audit [DBG] from='mgr.? 172.21.15.66:0/3940103742' entity='mgr.smithi066.aazujw' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi066.aazujw/key"}]: dispatch 2023-11-17T05:36:26.518 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:36:19 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:36:19.827365+0000 mon.smithi040 (mon.0) 379 : audit [DBG] from='mgr.? 172.21.15.66:0/3940103742' entity='mgr.smithi066.aazujw' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-11-17T05:36:26.518 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:36:20 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:36:19.885592+0000 mon.smithi040 (mon.0) 380 : cluster [DBG] mgrmap e18: smithi040.ogcprl(active, since 15m), standbys: smithi066.aazujw 2023-11-17T05:36:26.518 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:36:20 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:36:19.885784+0000 mon.smithi040 (mon.0) 381 : audit [DBG] from='mgr.14164 172.21.15.40:0/2203032451' entity='mgr.smithi040.ogcprl' cmd=[{"prefix": "mgr metadata", "who": "smithi066.aazujw", "id": "smithi066.aazujw"}]: dispatch 2023-11-17T05:36:26.518 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:36:20 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:36:20.787914+0000 mon.smithi040 (mon.0) 382 : audit [DBG] from='mgr.14164 172.21.15.40:0/2203032451' entity='mgr.smithi040.ogcprl' cmd=[{"prefix": "mon metadata", "id": "smithi066"}]: dispatch 2023-11-17T05:36:26.518 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:36:21 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:36:21.566301+0000 mgr.smithi040.ogcprl (mgr.14164) 471 : cluster [DBG] pgmap v450: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:36:26.518 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:36:23 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:36:23.566632+0000 mgr.smithi040.ogcprl (mgr.14164) 472 : cluster [DBG] pgmap v451: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:36:26.518 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:36:25 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:36:25.566949+0000 mgr.smithi040.ogcprl (mgr.14164) 473 : cluster [DBG] pgmap v452: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:36:27.495 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:36:27.494+0000 7fec1e425700 1 -- 172.21.15.66:0/316809159 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fec180f5410 msgr2=0x7fec180f5830 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:36:27.495 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:36:27.494+0000 7fec1e425700 1 --2- 172.21.15.66:0/316809159 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fec180f5410 0x7fec180f5830 secure :-1 s=READY pgs=455 cs=0 l=1 rev1=1 rx=0x7fec08009fb0 tx=0x7fec08009b30).stop 2023-11-17T05:36:27.495 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:36:27.494+0000 7fec1e425700 1 -- 172.21.15.66:0/316809159 shutdown_connections 2023-11-17T05:36:27.495 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:36:27.494+0000 7fec1e425700 1 --2- 172.21.15.66:0/316809159 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fec180f5410 0x7fec180f5830 unknown :-1 s=CLOSED pgs=455 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:36:27.496 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:36:27.494+0000 7fec1e425700 1 -- 172.21.15.66:0/316809159 >> 172.21.15.66:0/316809159 conn(0x7fec180f0b70 msgr2=0x7fec180f2f90 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:36:27.496 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:36:27.494+0000 7fec1e425700 1 -- 172.21.15.66:0/316809159 shutdown_connections 2023-11-17T05:36:27.496 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:36:27.494+0000 7fec1e425700 1 -- 172.21.15.66:0/316809159 wait complete. 2023-11-17T05:36:27.496 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:36:27.495+0000 7fec1e425700 1 Processor -- start 2023-11-17T05:36:27.496 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:36:27.495+0000 7fec1e425700 1 -- start start 2023-11-17T05:36:27.496 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:36:27.495+0000 7fec1e425700 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fec180f5410 0x7fec180f3110 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:36:27.496 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:36:27.495+0000 7fec1e425700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fec180f1590 0x7fec180f1a10 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:36:27.496 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:36:27.495+0000 7fec1e425700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fec180f1f90 con 0x7fec180f1590 2023-11-17T05:36:27.496 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:36:27.495+0000 7fec1e425700 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7fec180f20d0 con 0x7fec180f5410 2023-11-17T05:36:27.497 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:36:27.495+0000 7fec17fff700 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fec180f5410 0x7fec180f3110 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:36:27.497 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:36:27.495+0000 7fec17fff700 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fec180f5410 0x7fec180f3110 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 rx=0 tx=0).handle_hello peer v2:172.21.15.66:3300/0 says I am v2:172.21.15.66:50416/0 (socket says 172.21.15.66:50416) 2023-11-17T05:36:27.498 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:36:27.495+0000 7fec17fff700 1 -- 172.21.15.66:0/1798868291 learned_addr learned my addr 172.21.15.66:0/1798868291 (peer_addr_for_me v2:172.21.15.66:0/0) 2023-11-17T05:36:27.498 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:36:27.495+0000 7fec177fe700 1 --2- 172.21.15.66:0/1798868291 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fec180f1590 0x7fec180f1a10 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:36:27.498 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:36:27.496+0000 7fec17fff700 1 -- 172.21.15.66:0/1798868291 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fec180f1590 msgr2=0x7fec180f1a10 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:36:27.498 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:36:27.496+0000 7fec17fff700 1 --2- 172.21.15.66:0/1798868291 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fec180f1590 0x7fec180f1a10 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:36:27.498 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:36:27.496+0000 7fec17fff700 1 -- 172.21.15.66:0/1798868291 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7fec08005040 con 0x7fec180f5410 2023-11-17T05:36:27.498 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:36:27.496+0000 7fec17fff700 1 --2- 172.21.15.66:0/1798868291 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fec180f5410 0x7fec180f3110 secure :-1 s=READY pgs=3 cs=0 l=1 rev1=1 rx=0x7fec08000c00 tx=0x7fec0800b040).ready entity=mon.1 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:36:27.498 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:36:27.496+0000 7fec14ff9700 1 -- 172.21.15.66:0/1798868291 <== mon.1 v2:172.21.15.66:3300/0 1 ==== mon_map magic: 0 v1 ==== 357+0+0 (secure 0 0 0) 0x7fec08019070 con 0x7fec180f5410 2023-11-17T05:36:27.498 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:36:27.496+0000 7fec14ff9700 1 -- 172.21.15.66:0/1798868291 <== mon.1 v2:172.21.15.66:3300/0 2 ==== config(25 keys) v1 ==== 1074+0+0 (secure 0 0 0) 0x7fec08007ba0 con 0x7fec180f5410 2023-11-17T05:36:27.498 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:36:27.496+0000 7fec1e425700 1 -- 172.21.15.66:0/1798868291 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7fec180f2320 con 0x7fec180f5410 2023-11-17T05:36:27.498 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:36:27.496+0000 7fec14ff9700 1 -- 172.21.15.66:0/1798868291 <== mon.1 v2:172.21.15.66:3300/0 3 ==== mon_map magic: 0 v1 ==== 357+0+0 (secure 0 0 0) 0x7fec0801d3c0 con 0x7fec180f5410 2023-11-17T05:36:27.498 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:36:27.496+0000 7fec1e425700 1 -- 172.21.15.66:0/1798868291 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7fec1810ead0 con 0x7fec180f5410 2023-11-17T05:36:27.498 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:36:27.497+0000 7fec1e425700 1 -- 172.21.15.66:0/1798868291 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7fec18045ad0 con 0x7fec180f5410 2023-11-17T05:36:27.499 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:36:27.497+0000 7fec14ff9700 1 -- 172.21.15.66:0/1798868291 <== mon.1 v2:172.21.15.66:3300/0 4 ==== mgrmap(e 18) v1 ==== 72193+0+0 (secure 0 0 0) 0x7fec08025070 con 0x7fec180f5410 2023-11-17T05:36:27.499 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:36:27.498+0000 7fec14ff9700 1 --2- 172.21.15.66:0/1798868291 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fec0005c3b0 0x7fec0005e870 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:36:27.499 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:36:27.498+0000 7fec14ff9700 1 -- 172.21.15.66:0/1798868291 <== mon.1 v2:172.21.15.66:3300/0 5 ==== osd_map(5..5 src has 1..5) v4 ==== 1270+0+0 (secure 0 0 0) 0x7fec08076100 con 0x7fec180f5410 2023-11-17T05:36:27.499 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:36:27.498+0000 7fec177fe700 1 --2- 172.21.15.66:0/1798868291 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fec0005c3b0 0x7fec0005e870 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:36:27.499 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:36:27.498+0000 7fec177fe700 1 --2- 172.21.15.66:0/1798868291 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fec0005c3b0 0x7fec0005e870 secure :-1 s=READY pgs=206 cs=0 l=1 rev1=1 rx=0x7fec0c00b9f0 tx=0x7fec0c006e20).ready entity=mgr.14164 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:36:27.502 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:36:27.502+0000 7fec14ff9700 1 -- 172.21.15.66:0/1798868291 <== mon.1 v2:172.21.15.66:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+152212 (secure 0 0 0) 0x7fec08049dd0 con 0x7fec180f5410 2023-11-17T05:36:27.724 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:36:27.723+0000 7fec1e425700 1 -- 172.21.15.66:0/1798868291 --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_command({"prefix": "mon dump", "format": "json"} v 0) v1 -- 0x7fec18027a00 con 0x7fec180f5410 2023-11-17T05:36:27.725 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:36:27.724+0000 7fec14ff9700 1 -- 172.21.15.66:0/1798868291 <== mon.1 v2:172.21.15.66:3300/0 7 ==== mon_command_ack([{"prefix": "mon dump", "format": "json"}]=0 dumped monmap epoch 2 v2) v1 ==== 95+0+964 (secure 0 0 0) 0x7fec0804d3f0 con 0x7fec180f5410 2023-11-17T05:36:27.725 INFO:teuthology.orchestra.run.smithi066.stdout: 2023-11-17T05:36:27.725 INFO:teuthology.orchestra.run.smithi066.stdout:{"epoch":2,"fsid":"ed6470e2-8508-11ee-8dbe-212e2dc638e7","modified":"2023-11-17T05:36:14.787536Z","created":"2023-11-17T05:20:13.934038Z","min_mon_release":16,"min_mon_release_name":"pacific","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging"],"optional":[]},"mons":[{"rank":0,"name":"smithi040","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.40:3300","nonce":0},{"type":"v1","addr":"172.21.15.40:6789","nonce":0}]},"addr":"172.21.15.40:6789/0","public_addr":"172.21.15.40:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":1,"name":"smithi066","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.66:3300","nonce":0},{"type":"v1","addr":"172.21.15.66:6789","nonce":0}]},"addr":"172.21.15.66:6789/0","public_addr":"172.21.15.66:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0,1]} 2023-11-17T05:36:27.726 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:36:27.725+0000 7fec1e425700 1 -- 172.21.15.66:0/1798868291 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fec0005c3b0 msgr2=0x7fec0005e870 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:36:27.726 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:36:27.725+0000 7fec1e425700 1 --2- 172.21.15.66:0/1798868291 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fec0005c3b0 0x7fec0005e870 secure :-1 s=READY pgs=206 cs=0 l=1 rev1=1 rx=0x7fec0c00b9f0 tx=0x7fec0c006e20).stop 2023-11-17T05:36:27.726 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:36:27.725+0000 7fec1e425700 1 -- 172.21.15.66:0/1798868291 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fec180f5410 msgr2=0x7fec180f3110 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:36:27.727 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:36:27.725+0000 7fec1e425700 1 --2- 172.21.15.66:0/1798868291 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fec180f5410 0x7fec180f3110 secure :-1 s=READY pgs=3 cs=0 l=1 rev1=1 rx=0x7fec08000c00 tx=0x7fec0800b040).stop 2023-11-17T05:36:27.727 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:36:27.725+0000 7fec1e425700 1 -- 172.21.15.66:0/1798868291 shutdown_connections 2023-11-17T05:36:27.727 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:36:27.725+0000 7fec1e425700 1 --2- 172.21.15.66:0/1798868291 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7fec0005c3b0 0x7fec0005e870 unknown :-1 s=CLOSED pgs=206 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:36:27.727 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:36:27.725+0000 7fec1e425700 1 --2- 172.21.15.66:0/1798868291 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7fec180f5410 0x7fec180f3110 unknown :-1 s=CLOSED pgs=3 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:36:27.727 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:36:27.725+0000 7fec1e425700 1 --2- 172.21.15.66:0/1798868291 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7fec180f1590 0x7fec180f1a10 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:36:27.727 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:36:27.726+0000 7fec1e425700 1 -- 172.21.15.66:0/1798868291 >> 172.21.15.66:0/1798868291 conn(0x7fec180f0b70 msgr2=0x7fec1805fe40 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:36:27.727 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:36:27.726+0000 7fec1e425700 1 -- 172.21.15.66:0/1798868291 shutdown_connections 2023-11-17T05:36:27.727 INFO:teuthology.orchestra.run.smithi066.stderr:2023-11-17T05:36:27.726+0000 7fec1e425700 1 -- 172.21.15.66:0/1798868291 wait complete. 2023-11-17T05:36:27.727 INFO:teuthology.orchestra.run.smithi066.stderr:dumped monmap epoch 2 2023-11-17T05:36:27.931 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:27 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: cluster 2023-11-17T05:36:27.567223+0000 mgr.smithi040.ogcprl (mgr.14164) 474 : cluster [DBG] pgmap v453: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:36:27.989 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:36:27 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:36:27.567223+0000 mgr.smithi040.ogcprl (mgr.14164) 474 : cluster [DBG] pgmap v453: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:36:28.145 INFO:tasks.cephadm:Generating final ceph.conf file... 2023-11-17T05:36:28.145 DEBUG:teuthology.orchestra.run.smithi040:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.4 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ed6470e2-8508-11ee-8dbe-212e2dc638e7 -- ceph config generate-minimal-conf 2023-11-17T05:36:28.741 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:36:28 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:36:27.724629+0000 mon.smithi066 (mon.1) 2 : audit [DBG] from='client.? 172.21.15.66:0/1798868291' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-11-17T05:36:29.049 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:28 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: audit 2023-11-17T05:36:27.724629+0000 mon.smithi066 (mon.1) 2 : audit [DBG] from='client.? 172.21.15.66:0/1798868291' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-11-17T05:36:29.396 INFO:teuthology.orchestra.run.smithi040.stderr:2023-11-17T05:36:29.395+0000 7f34751dd700 1 -- 172.21.15.40:0/1559353362 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f34700f6510 msgr2=0x7f34700f6930 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:36:29.397 INFO:teuthology.orchestra.run.smithi040.stderr:2023-11-17T05:36:29.395+0000 7f34751dd700 1 --2- 172.21.15.40:0/1559353362 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f34700f6510 0x7f34700f6930 secure :-1 s=READY pgs=456 cs=0 l=1 rev1=1 rx=0x7f3460009fb0 tx=0x7f3460009b30).stop 2023-11-17T05:36:29.397 INFO:teuthology.orchestra.run.smithi040.stderr:2023-11-17T05:36:29.395+0000 7f34751dd700 1 -- 172.21.15.40:0/1559353362 shutdown_connections 2023-11-17T05:36:29.397 INFO:teuthology.orchestra.run.smithi040.stderr:2023-11-17T05:36:29.395+0000 7f34751dd700 1 --2- 172.21.15.40:0/1559353362 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f34700f6510 0x7f34700f6930 unknown :-1 s=CLOSED pgs=456 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:36:29.397 INFO:teuthology.orchestra.run.smithi040.stderr:2023-11-17T05:36:29.395+0000 7f34751dd700 1 -- 172.21.15.40:0/1559353362 >> 172.21.15.40:0/1559353362 conn(0x7f34700f1c70 msgr2=0x7f34700f4090 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:36:29.397 INFO:teuthology.orchestra.run.smithi040.stderr:2023-11-17T05:36:29.395+0000 7f34751dd700 1 -- 172.21.15.40:0/1559353362 shutdown_connections 2023-11-17T05:36:29.397 INFO:teuthology.orchestra.run.smithi040.stderr:2023-11-17T05:36:29.395+0000 7f34751dd700 1 -- 172.21.15.40:0/1559353362 wait complete. 2023-11-17T05:36:29.399 INFO:teuthology.orchestra.run.smithi040.stderr:2023-11-17T05:36:29.396+0000 7f34751dd700 1 Processor -- start 2023-11-17T05:36:29.399 INFO:teuthology.orchestra.run.smithi040.stderr:2023-11-17T05:36:29.396+0000 7f34751dd700 1 -- start start 2023-11-17T05:36:29.399 INFO:teuthology.orchestra.run.smithi040.stderr:2023-11-17T05:36:29.396+0000 7f34751dd700 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f34700f6510 0x7f3470106770 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:36:29.399 INFO:teuthology.orchestra.run.smithi040.stderr:2023-11-17T05:36:29.396+0000 7f34751dd700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f3470106c60 0x7f3470107b50 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:36:29.399 INFO:teuthology.orchestra.run.smithi040.stderr:2023-11-17T05:36:29.396+0000 7f34751dd700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f34701071c0 con 0x7f3470106c60 2023-11-17T05:36:29.399 INFO:teuthology.orchestra.run.smithi040.stderr:2023-11-17T05:36:29.396+0000 7f34751dd700 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f3470107300 con 0x7f34700f6510 2023-11-17T05:36:29.399 INFO:teuthology.orchestra.run.smithi040.stderr:2023-11-17T05:36:29.396+0000 7f346e59c700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f3470106c60 0x7f3470107b50 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:36:29.399 INFO:teuthology.orchestra.run.smithi040.stderr:2023-11-17T05:36:29.396+0000 7f346e59c700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f3470106c60 0x7f3470107b50 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:53120/0 (socket says 172.21.15.40:53120) 2023-11-17T05:36:29.399 INFO:teuthology.orchestra.run.smithi040.stderr:2023-11-17T05:36:29.396+0000 7f346e59c700 1 -- 172.21.15.40:0/480642366 learned_addr learned my addr 172.21.15.40:0/480642366 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-11-17T05:36:29.399 INFO:teuthology.orchestra.run.smithi040.stderr:2023-11-17T05:36:29.396+0000 7f346ed9d700 1 --2- 172.21.15.40:0/480642366 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f34700f6510 0x7f3470106770 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:36:29.400 INFO:teuthology.orchestra.run.smithi040.stderr:2023-11-17T05:36:29.396+0000 7f346e59c700 1 -- 172.21.15.40:0/480642366 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f34700f6510 msgr2=0x7f3470106770 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:36:29.400 INFO:teuthology.orchestra.run.smithi040.stderr:2023-11-17T05:36:29.396+0000 7f346e59c700 1 --2- 172.21.15.40:0/480642366 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f34700f6510 0x7f3470106770 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:36:29.400 INFO:teuthology.orchestra.run.smithi040.stderr:2023-11-17T05:36:29.396+0000 7f346e59c700 1 -- 172.21.15.40:0/480642366 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f3460005040 con 0x7f3470106c60 2023-11-17T05:36:29.400 INFO:teuthology.orchestra.run.smithi040.stderr:2023-11-17T05:36:29.397+0000 7f346e59c700 1 --2- 172.21.15.40:0/480642366 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f3470106c60 0x7f3470107b50 secure :-1 s=READY pgs=457 cs=0 l=1 rev1=1 rx=0x7f34640093c0 tx=0x7f3464006ee0).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:36:29.400 INFO:teuthology.orchestra.run.smithi040.stderr:2023-11-17T05:36:29.397+0000 7f34577fe700 1 -- 172.21.15.40:0/480642366 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 357+0+0 (secure 0 0 0) 0x7f346400e650 con 0x7f3470106c60 2023-11-17T05:36:29.400 INFO:teuthology.orchestra.run.smithi040.stderr:2023-11-17T05:36:29.397+0000 7f34577fe700 1 -- 172.21.15.40:0/480642366 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(25 keys) v1 ==== 1074+0+0 (secure 0 0 0) 0x7f3464009e70 con 0x7f3470106c60 2023-11-17T05:36:29.400 INFO:teuthology.orchestra.run.smithi040.stderr:2023-11-17T05:36:29.397+0000 7f34751dd700 1 -- 172.21.15.40:0/480642366 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f3470108160 con 0x7f3470106c60 2023-11-17T05:36:29.400 INFO:teuthology.orchestra.run.smithi040.stderr:2023-11-17T05:36:29.397+0000 7f34577fe700 1 -- 172.21.15.40:0/480642366 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 357+0+0 (secure 0 0 0) 0x7f3464011c30 con 0x7f3470106c60 2023-11-17T05:36:29.400 INFO:teuthology.orchestra.run.smithi040.stderr:2023-11-17T05:36:29.397+0000 7f34751dd700 1 -- 172.21.15.40:0/480642366 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f3470108720 con 0x7f3470106c60 2023-11-17T05:36:29.401 INFO:teuthology.orchestra.run.smithi040.stderr:2023-11-17T05:36:29.398+0000 7f34751dd700 1 -- 172.21.15.40:0/480642366 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f3470059100 con 0x7f3470106c60 2023-11-17T05:36:29.401 INFO:teuthology.orchestra.run.smithi040.stderr:2023-11-17T05:36:29.398+0000 7f34577fe700 1 -- 172.21.15.40:0/480642366 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 18) v1 ==== 72193+0+0 (secure 0 0 0) 0x7f3464018030 con 0x7f3470106c60 2023-11-17T05:36:29.401 INFO:teuthology.orchestra.run.smithi040.stderr:2023-11-17T05:36:29.399+0000 7f34577fe700 1 --2- 172.21.15.40:0/480642366 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f345805c3b0 0x7f345805e870 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:36:29.401 INFO:teuthology.orchestra.run.smithi040.stderr:2023-11-17T05:36:29.399+0000 7f34577fe700 1 -- 172.21.15.40:0/480642366 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(5..5 src has 1..5) v4 ==== 1270+0+0 (secure 0 0 0) 0x7f34640749c0 con 0x7f3470106c60 2023-11-17T05:36:29.401 INFO:teuthology.orchestra.run.smithi040.stderr:2023-11-17T05:36:29.399+0000 7f346ed9d700 1 --2- 172.21.15.40:0/480642366 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f345805c3b0 0x7f345805e870 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:36:29.401 INFO:teuthology.orchestra.run.smithi040.stderr:2023-11-17T05:36:29.399+0000 7f346ed9d700 1 --2- 172.21.15.40:0/480642366 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f345805c3b0 0x7f345805e870 secure :-1 s=READY pgs=207 cs=0 l=1 rev1=1 rx=0x7f34600043d0 tx=0x7f346000b040).ready entity=mgr.14164 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:36:29.405 INFO:teuthology.orchestra.run.smithi040.stderr:2023-11-17T05:36:29.403+0000 7f34577fe700 1 -- 172.21.15.40:0/480642366 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+152212 (secure 0 0 0) 0x7f34640452b0 con 0x7f3470106c60 2023-11-17T05:36:29.574 INFO:teuthology.orchestra.run.smithi040.stderr:2023-11-17T05:36:29.572+0000 7f34751dd700 1 -- 172.21.15.40:0/480642366 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "config generate-minimal-conf"} v 0) v1 -- 0x7f347010d250 con 0x7f3470106c60 2023-11-17T05:36:29.574 INFO:teuthology.orchestra.run.smithi040.stderr:2023-11-17T05:36:29.573+0000 7f34577fe700 1 -- 172.21.15.40:0/480642366 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mon_command_ack([{"prefix": "config generate-minimal-conf"}]=0 v12) v1 ==== 76+0+223 (secure 0 0 0) 0x7f3464016020 con 0x7f3470106c60 2023-11-17T05:36:29.574 INFO:teuthology.orchestra.run.smithi040.stdout:# minimal ceph.conf for ed6470e2-8508-11ee-8dbe-212e2dc638e7 2023-11-17T05:36:29.574 INFO:teuthology.orchestra.run.smithi040.stdout:[global] 2023-11-17T05:36:29.575 INFO:teuthology.orchestra.run.smithi040.stdout: fsid = ed6470e2-8508-11ee-8dbe-212e2dc638e7 2023-11-17T05:36:29.575 INFO:teuthology.orchestra.run.smithi040.stdout: mon_host = [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] 2023-11-17T05:36:29.576 INFO:teuthology.orchestra.run.smithi040.stderr:2023-11-17T05:36:29.575+0000 7f34751dd700 1 -- 172.21.15.40:0/480642366 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f345805c3b0 msgr2=0x7f345805e870 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:36:29.576 INFO:teuthology.orchestra.run.smithi040.stderr:2023-11-17T05:36:29.575+0000 7f34751dd700 1 --2- 172.21.15.40:0/480642366 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f345805c3b0 0x7f345805e870 secure :-1 s=READY pgs=207 cs=0 l=1 rev1=1 rx=0x7f34600043d0 tx=0x7f346000b040).stop 2023-11-17T05:36:29.576 INFO:teuthology.orchestra.run.smithi040.stderr:2023-11-17T05:36:29.575+0000 7f34751dd700 1 -- 172.21.15.40:0/480642366 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f3470106c60 msgr2=0x7f3470107b50 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:36:29.577 INFO:teuthology.orchestra.run.smithi040.stderr:2023-11-17T05:36:29.575+0000 7f34751dd700 1 --2- 172.21.15.40:0/480642366 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f3470106c60 0x7f3470107b50 secure :-1 s=READY pgs=457 cs=0 l=1 rev1=1 rx=0x7f34640093c0 tx=0x7f3464006ee0).stop 2023-11-17T05:36:29.577 INFO:teuthology.orchestra.run.smithi040.stderr:2023-11-17T05:36:29.575+0000 7f34751dd700 1 -- 172.21.15.40:0/480642366 shutdown_connections 2023-11-17T05:36:29.577 INFO:teuthology.orchestra.run.smithi040.stderr:2023-11-17T05:36:29.575+0000 7f34751dd700 1 --2- 172.21.15.40:0/480642366 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f345805c3b0 0x7f345805e870 unknown :-1 s=CLOSED pgs=207 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:36:29.577 INFO:teuthology.orchestra.run.smithi040.stderr:2023-11-17T05:36:29.575+0000 7f34751dd700 1 --2- 172.21.15.40:0/480642366 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f34700f6510 0x7f3470106770 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:36:29.577 INFO:teuthology.orchestra.run.smithi040.stderr:2023-11-17T05:36:29.575+0000 7f34751dd700 1 --2- 172.21.15.40:0/480642366 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f3470106c60 0x7f3470107b50 unknown :-1 s=CLOSED pgs=457 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:36:29.577 INFO:teuthology.orchestra.run.smithi040.stderr:2023-11-17T05:36:29.575+0000 7f34751dd700 1 -- 172.21.15.40:0/480642366 >> 172.21.15.40:0/480642366 conn(0x7f34700f1c70 msgr2=0x7f34700f26a0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:36:29.577 INFO:teuthology.orchestra.run.smithi040.stderr:2023-11-17T05:36:29.575+0000 7f34751dd700 1 -- 172.21.15.40:0/480642366 shutdown_connections 2023-11-17T05:36:29.577 INFO:teuthology.orchestra.run.smithi040.stderr:2023-11-17T05:36:29.575+0000 7f34751dd700 1 -- 172.21.15.40:0/480642366 wait complete. 2023-11-17T05:36:30.000 INFO:tasks.cephadm:Distributing (final) config and client.admin keyring... 2023-11-17T05:36:30.000 DEBUG:teuthology.orchestra.run.smithi040:> set -ex 2023-11-17T05:36:30.000 DEBUG:teuthology.orchestra.run.smithi040:> sudo dd of=/etc/ceph/ceph.conf 2023-11-17T05:36:30.039 DEBUG:teuthology.orchestra.run.smithi040:> set -ex 2023-11-17T05:36:30.039 DEBUG:teuthology.orchestra.run.smithi040:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2023-11-17T05:36:30.114 DEBUG:teuthology.orchestra.run.smithi066:> set -ex 2023-11-17T05:36:30.114 DEBUG:teuthology.orchestra.run.smithi066:> sudo dd of=/etc/ceph/ceph.conf 2023-11-17T05:36:30.153 DEBUG:teuthology.orchestra.run.smithi066:> set -ex 2023-11-17T05:36:30.153 DEBUG:teuthology.orchestra.run.smithi066:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2023-11-17T05:36:30.227 INFO:tasks.cephadm:Deploying OSDs... 2023-11-17T05:36:30.227 DEBUG:teuthology.orchestra.run.smithi040:> set -ex 2023-11-17T05:36:30.227 DEBUG:teuthology.orchestra.run.smithi040:> dd if=/scratch_devs of=/dev/stdout 2023-11-17T05:36:30.250 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2023-11-17T05:36:30.251 DEBUG:teuthology.orchestra.run.smithi040:> stat /dev/vg_nvme/lv_1 2023-11-17T05:36:30.313 INFO:teuthology.orchestra.run.smithi040.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2023-11-17T05:36:30.313 INFO:teuthology.orchestra.run.smithi040.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-11-17T05:36:30.313 INFO:teuthology.orchestra.run.smithi040.stdout:Device: 6h/6d Inode: 391556 Links: 1 2023-11-17T05:36:30.313 INFO:teuthology.orchestra.run.smithi040.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-11-17T05:36:30.313 INFO:teuthology.orchestra.run.smithi040.stdout:Context: system_u:object_r:device_t:s0 2023-11-17T05:36:30.313 INFO:teuthology.orchestra.run.smithi040.stdout:Access: 2023-11-17 05:26:40.224731847 +0000 2023-11-17T05:36:30.314 INFO:teuthology.orchestra.run.smithi040.stdout:Modify: 2023-11-17 05:12:33.263886036 +0000 2023-11-17T05:36:30.314 INFO:teuthology.orchestra.run.smithi040.stdout:Change: 2023-11-17 05:12:33.263886036 +0000 2023-11-17T05:36:30.314 INFO:teuthology.orchestra.run.smithi040.stdout: Birth: - 2023-11-17T05:36:30.314 DEBUG:teuthology.orchestra.run.smithi040:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2023-11-17T05:36:30.385 INFO:teuthology.orchestra.run.smithi040.stderr:1+0 records in 2023-11-17T05:36:30.386 INFO:teuthology.orchestra.run.smithi040.stderr:1+0 records out 2023-11-17T05:36:30.386 INFO:teuthology.orchestra.run.smithi040.stderr:512 bytes copied, 0.00024411 s, 2.1 MB/s 2023-11-17T05:36:30.387 DEBUG:teuthology.orchestra.run.smithi040:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2023-11-17T05:36:30.452 DEBUG:teuthology.orchestra.run.smithi040:> stat /dev/vg_nvme/lv_2 2023-11-17T05:36:30.516 INFO:teuthology.orchestra.run.smithi040.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2023-11-17T05:36:30.516 INFO:teuthology.orchestra.run.smithi040.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-11-17T05:36:30.516 INFO:teuthology.orchestra.run.smithi040.stdout:Device: 6h/6d Inode: 393712 Links: 1 2023-11-17T05:36:30.516 INFO:teuthology.orchestra.run.smithi040.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-11-17T05:36:30.516 INFO:teuthology.orchestra.run.smithi040.stdout:Context: system_u:object_r:device_t:s0 2023-11-17T05:36:30.517 INFO:teuthology.orchestra.run.smithi040.stdout:Access: 2023-11-17 05:26:40.224731847 +0000 2023-11-17T05:36:30.517 INFO:teuthology.orchestra.run.smithi040.stdout:Modify: 2023-11-17 05:12:33.888877937 +0000 2023-11-17T05:36:30.517 INFO:teuthology.orchestra.run.smithi040.stdout:Change: 2023-11-17 05:12:33.888877937 +0000 2023-11-17T05:36:30.517 INFO:teuthology.orchestra.run.smithi040.stdout: Birth: - 2023-11-17T05:36:30.517 DEBUG:teuthology.orchestra.run.smithi040:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2023-11-17T05:36:30.590 INFO:teuthology.orchestra.run.smithi040.stderr:1+0 records in 2023-11-17T05:36:30.590 INFO:teuthology.orchestra.run.smithi040.stderr:1+0 records out 2023-11-17T05:36:30.590 INFO:teuthology.orchestra.run.smithi040.stderr:512 bytes copied, 0.000236809 s, 2.2 MB/s 2023-11-17T05:36:30.591 DEBUG:teuthology.orchestra.run.smithi040:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2023-11-17T05:36:30.656 DEBUG:teuthology.orchestra.run.smithi040:> stat /dev/vg_nvme/lv_3 2023-11-17T05:36:30.720 INFO:teuthology.orchestra.run.smithi040.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2023-11-17T05:36:30.721 INFO:teuthology.orchestra.run.smithi040.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-11-17T05:36:30.721 INFO:teuthology.orchestra.run.smithi040.stdout:Device: 6h/6d Inode: 393819 Links: 1 2023-11-17T05:36:30.721 INFO:teuthology.orchestra.run.smithi040.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-11-17T05:36:30.721 INFO:teuthology.orchestra.run.smithi040.stdout:Context: system_u:object_r:device_t:s0 2023-11-17T05:36:30.721 INFO:teuthology.orchestra.run.smithi040.stdout:Access: 2023-11-17 05:26:40.224731847 +0000 2023-11-17T05:36:30.721 INFO:teuthology.orchestra.run.smithi040.stdout:Modify: 2023-11-17 05:12:34.530869619 +0000 2023-11-17T05:36:30.722 INFO:teuthology.orchestra.run.smithi040.stdout:Change: 2023-11-17 05:12:34.530869619 +0000 2023-11-17T05:36:30.722 INFO:teuthology.orchestra.run.smithi040.stdout: Birth: - 2023-11-17T05:36:30.722 DEBUG:teuthology.orchestra.run.smithi040:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2023-11-17T05:36:30.739 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:36:30 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:36:29.567467+0000 mgr.smithi040.ogcprl (mgr.14164) 475 : cluster [DBG] pgmap v454: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:36:30.739 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:36:30 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:36:29.574622+0000 mon.smithi040 (mon.0) 383 : audit [DBG] from='client.? 172.21.15.40:0/480642366' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-11-17T05:36:30.772 INFO:teuthology.orchestra.run.smithi040.stderr:1+0 records in 2023-11-17T05:36:30.773 INFO:teuthology.orchestra.run.smithi040.stderr:1+0 records out 2023-11-17T05:36:30.773 INFO:teuthology.orchestra.run.smithi040.stderr:512 bytes copied, 0.00022549 s, 2.3 MB/s 2023-11-17T05:36:30.774 DEBUG:teuthology.orchestra.run.smithi040:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2023-11-17T05:36:30.800 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:30 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: cluster 2023-11-17T05:36:29.567467+0000 mgr.smithi040.ogcprl (mgr.14164) 475 : cluster [DBG] pgmap v454: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:36:30.800 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:30 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: audit 2023-11-17T05:36:29.574622+0000 mon.smithi040 (mon.0) 383 : audit [DBG] from='client.? 172.21.15.40:0/480642366' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-11-17T05:36:30.838 DEBUG:teuthology.orchestra.run.smithi040:> stat /dev/vg_nvme/lv_4 2023-11-17T05:36:30.903 INFO:teuthology.orchestra.run.smithi040.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2023-11-17T05:36:30.903 INFO:teuthology.orchestra.run.smithi040.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-11-17T05:36:30.903 INFO:teuthology.orchestra.run.smithi040.stdout:Device: 6h/6d Inode: 393897 Links: 1 2023-11-17T05:36:30.903 INFO:teuthology.orchestra.run.smithi040.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-11-17T05:36:30.903 INFO:teuthology.orchestra.run.smithi040.stdout:Context: system_u:object_r:device_t:s0 2023-11-17T05:36:30.903 INFO:teuthology.orchestra.run.smithi040.stdout:Access: 2023-11-17 05:26:40.225731833 +0000 2023-11-17T05:36:30.903 INFO:teuthology.orchestra.run.smithi040.stdout:Modify: 2023-11-17 05:12:35.105862168 +0000 2023-11-17T05:36:30.903 INFO:teuthology.orchestra.run.smithi040.stdout:Change: 2023-11-17 05:12:35.105862168 +0000 2023-11-17T05:36:30.903 INFO:teuthology.orchestra.run.smithi040.stdout: Birth: - 2023-11-17T05:36:30.904 DEBUG:teuthology.orchestra.run.smithi040:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2023-11-17T05:36:30.976 INFO:teuthology.orchestra.run.smithi040.stderr:1+0 records in 2023-11-17T05:36:30.976 INFO:teuthology.orchestra.run.smithi040.stderr:1+0 records out 2023-11-17T05:36:30.976 INFO:teuthology.orchestra.run.smithi040.stderr:512 bytes copied, 0.000212407 s, 2.4 MB/s 2023-11-17T05:36:30.977 DEBUG:teuthology.orchestra.run.smithi040:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2023-11-17T05:36:31.042 DEBUG:teuthology.orchestra.run.smithi066:> set -ex 2023-11-17T05:36:31.042 DEBUG:teuthology.orchestra.run.smithi066:> dd if=/scratch_devs of=/dev/stdout 2023-11-17T05:36:31.065 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2023-11-17T05:36:31.065 DEBUG:teuthology.orchestra.run.smithi066:> stat /dev/vg_nvme/lv_1 2023-11-17T05:36:31.127 INFO:teuthology.orchestra.run.smithi066.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2023-11-17T05:36:31.127 INFO:teuthology.orchestra.run.smithi066.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-11-17T05:36:31.127 INFO:teuthology.orchestra.run.smithi066.stdout:Device: 6h/6d Inode: 390982 Links: 1 2023-11-17T05:36:31.127 INFO:teuthology.orchestra.run.smithi066.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-11-17T05:36:31.127 INFO:teuthology.orchestra.run.smithi066.stdout:Context: system_u:object_r:device_t:s0 2023-11-17T05:36:31.128 INFO:teuthology.orchestra.run.smithi066.stdout:Access: 2023-11-17 05:36:15.090421127 +0000 2023-11-17T05:36:31.128 INFO:teuthology.orchestra.run.smithi066.stdout:Modify: 2023-11-17 05:12:04.138809607 +0000 2023-11-17T05:36:31.128 INFO:teuthology.orchestra.run.smithi066.stdout:Change: 2023-11-17 05:12:04.138809607 +0000 2023-11-17T05:36:31.128 INFO:teuthology.orchestra.run.smithi066.stdout: Birth: - 2023-11-17T05:36:31.128 DEBUG:teuthology.orchestra.run.smithi066:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2023-11-17T05:36:31.202 INFO:teuthology.orchestra.run.smithi066.stderr:1+0 records in 2023-11-17T05:36:31.202 INFO:teuthology.orchestra.run.smithi066.stderr:1+0 records out 2023-11-17T05:36:31.202 INFO:teuthology.orchestra.run.smithi066.stderr:512 bytes copied, 0.000262171 s, 2.0 MB/s 2023-11-17T05:36:31.202 DEBUG:teuthology.orchestra.run.smithi066:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2023-11-17T05:36:31.265 DEBUG:teuthology.orchestra.run.smithi066:> stat /dev/vg_nvme/lv_2 2023-11-17T05:36:31.325 INFO:teuthology.orchestra.run.smithi066.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2023-11-17T05:36:31.325 INFO:teuthology.orchestra.run.smithi066.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-11-17T05:36:31.325 INFO:teuthology.orchestra.run.smithi066.stdout:Device: 6h/6d Inode: 395511 Links: 1 2023-11-17T05:36:31.325 INFO:teuthology.orchestra.run.smithi066.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-11-17T05:36:31.326 INFO:teuthology.orchestra.run.smithi066.stdout:Context: system_u:object_r:device_t:s0 2023-11-17T05:36:31.326 INFO:teuthology.orchestra.run.smithi066.stdout:Access: 2023-11-17 05:36:15.091421119 +0000 2023-11-17T05:36:31.326 INFO:teuthology.orchestra.run.smithi066.stdout:Modify: 2023-11-17 05:12:04.716805174 +0000 2023-11-17T05:36:31.326 INFO:teuthology.orchestra.run.smithi066.stdout:Change: 2023-11-17 05:12:04.716805174 +0000 2023-11-17T05:36:31.326 INFO:teuthology.orchestra.run.smithi066.stdout: Birth: - 2023-11-17T05:36:31.326 DEBUG:teuthology.orchestra.run.smithi066:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2023-11-17T05:36:31.395 INFO:teuthology.orchestra.run.smithi066.stderr:1+0 records in 2023-11-17T05:36:31.395 INFO:teuthology.orchestra.run.smithi066.stderr:1+0 records out 2023-11-17T05:36:31.395 INFO:teuthology.orchestra.run.smithi066.stderr:512 bytes copied, 0.00023007 s, 2.2 MB/s 2023-11-17T05:36:31.396 DEBUG:teuthology.orchestra.run.smithi066:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2023-11-17T05:36:31.461 DEBUG:teuthology.orchestra.run.smithi066:> stat /dev/vg_nvme/lv_3 2023-11-17T05:36:31.521 INFO:teuthology.orchestra.run.smithi066.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2023-11-17T05:36:31.522 INFO:teuthology.orchestra.run.smithi066.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-11-17T05:36:31.522 INFO:teuthology.orchestra.run.smithi066.stdout:Device: 6h/6d Inode: 392746 Links: 1 2023-11-17T05:36:31.522 INFO:teuthology.orchestra.run.smithi066.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-11-17T05:36:31.522 INFO:teuthology.orchestra.run.smithi066.stdout:Context: system_u:object_r:device_t:s0 2023-11-17T05:36:31.522 INFO:teuthology.orchestra.run.smithi066.stdout:Access: 2023-11-17 05:36:15.091421119 +0000 2023-11-17T05:36:31.522 INFO:teuthology.orchestra.run.smithi066.stdout:Modify: 2023-11-17 05:12:05.929795867 +0000 2023-11-17T05:36:31.522 INFO:teuthology.orchestra.run.smithi066.stdout:Change: 2023-11-17 05:12:05.929795867 +0000 2023-11-17T05:36:31.522 INFO:teuthology.orchestra.run.smithi066.stdout: Birth: - 2023-11-17T05:36:31.522 DEBUG:teuthology.orchestra.run.smithi066:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2023-11-17T05:36:31.591 INFO:teuthology.orchestra.run.smithi066.stderr:1+0 records in 2023-11-17T05:36:31.591 INFO:teuthology.orchestra.run.smithi066.stderr:1+0 records out 2023-11-17T05:36:31.591 INFO:teuthology.orchestra.run.smithi066.stderr:512 bytes copied, 0.000224037 s, 2.3 MB/s 2023-11-17T05:36:31.592 DEBUG:teuthology.orchestra.run.smithi066:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2023-11-17T05:36:31.653 DEBUG:teuthology.orchestra.run.smithi066:> stat /dev/vg_nvme/lv_4 2023-11-17T05:36:31.714 INFO:teuthology.orchestra.run.smithi066.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2023-11-17T05:36:31.714 INFO:teuthology.orchestra.run.smithi066.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-11-17T05:36:31.714 INFO:teuthology.orchestra.run.smithi066.stdout:Device: 6h/6d Inode: 392834 Links: 1 2023-11-17T05:36:31.714 INFO:teuthology.orchestra.run.smithi066.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-11-17T05:36:31.715 INFO:teuthology.orchestra.run.smithi066.stdout:Context: system_u:object_r:device_t:s0 2023-11-17T05:36:31.715 INFO:teuthology.orchestra.run.smithi066.stdout:Access: 2023-11-17 05:36:15.091421119 +0000 2023-11-17T05:36:31.715 INFO:teuthology.orchestra.run.smithi066.stdout:Modify: 2023-11-17 05:12:06.467791741 +0000 2023-11-17T05:36:31.715 INFO:teuthology.orchestra.run.smithi066.stdout:Change: 2023-11-17 05:12:06.467791741 +0000 2023-11-17T05:36:31.715 INFO:teuthology.orchestra.run.smithi066.stdout: Birth: - 2023-11-17T05:36:31.715 DEBUG:teuthology.orchestra.run.smithi066:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2023-11-17T05:36:31.782 INFO:teuthology.orchestra.run.smithi066.stderr:1+0 records in 2023-11-17T05:36:31.782 INFO:teuthology.orchestra.run.smithi066.stderr:1+0 records out 2023-11-17T05:36:31.782 INFO:teuthology.orchestra.run.smithi066.stderr:512 bytes copied, 0.000221756 s, 2.3 MB/s 2023-11-17T05:36:31.783 DEBUG:teuthology.orchestra.run.smithi066:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2023-11-17T05:36:31.843 INFO:tasks.cephadm:Deploying osd.0 on smithi040 with /dev/vg_nvme/lv_4... 2023-11-17T05:36:31.843 DEBUG:teuthology.orchestra.run.smithi040:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.4 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ed6470e2-8508-11ee-8dbe-212e2dc638e7 -- ceph-volume lvm zap /dev/vg_nvme/lv_4 2023-11-17T05:36:31.988 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:36:31 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:36:31.567768+0000 mgr.smithi040.ogcprl (mgr.14164) 476 : cluster [DBG] pgmap v455: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:36:32.051 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:31 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: cluster 2023-11-17T05:36:31.567768+0000 mgr.smithi040.ogcprl (mgr.14164) 476 : cluster [DBG] pgmap v455: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:36:33.535 INFO:teuthology.orchestra.run.smithi040.stderr:--> Zapping: /dev/vg_nvme/lv_4 2023-11-17T05:36:33.535 INFO:teuthology.orchestra.run.smithi040.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_4 bs=1M count=10 conv=fsync 2023-11-17T05:36:33.535 INFO:teuthology.orchestra.run.smithi040.stderr: stderr: 10+0 records in 2023-11-17T05:36:33.535 INFO:teuthology.orchestra.run.smithi040.stderr:10+0 records out 2023-11-17T05:36:33.535 INFO:teuthology.orchestra.run.smithi040.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0114059 s, 919 MB/s 2023-11-17T05:36:33.535 INFO:teuthology.orchestra.run.smithi040.stderr:--> Zapping successful for: 2023-11-17T05:36:33.911 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:36:33 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:36:33.568012+0000 mgr.smithi040.ogcprl (mgr.14164) 477 : cluster [DBG] pgmap v456: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:36:33.943 DEBUG:teuthology.orchestra.run.smithi040:> sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.4 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ed6470e2-8508-11ee-8dbe-212e2dc638e7 -- ceph orch daemon add osd smithi040:vg_nvme/lv_4 2023-11-17T05:36:34.050 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:33 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: cluster 2023-11-17T05:36:33.568012+0000 mgr.smithi040.ogcprl (mgr.14164) 477 : cluster [DBG] pgmap v456: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:36:35.419 INFO:teuthology.orchestra.run.smithi040.stderr:2023-11-17T05:36:35.415+0000 7f1d927ec700 1 -- 172.21.15.40:0/2180632473 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f1d8c060310 msgr2=0x7f1d8c0fb510 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:36:35.419 INFO:teuthology.orchestra.run.smithi040.stderr:2023-11-17T05:36:35.415+0000 7f1d927ec700 1 --2- 172.21.15.40:0/2180632473 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f1d8c060310 0x7f1d8c0fb510 secure :-1 s=READY pgs=458 cs=0 l=1 rev1=1 rx=0x7f1d88004660 tx=0x7f1d88009b30).stop 2023-11-17T05:36:35.420 INFO:teuthology.orchestra.run.smithi040.stderr:2023-11-17T05:36:35.415+0000 7f1d927ec700 1 -- 172.21.15.40:0/2180632473 shutdown_connections 2023-11-17T05:36:35.420 INFO:teuthology.orchestra.run.smithi040.stderr:2023-11-17T05:36:35.415+0000 7f1d927ec700 1 --2- 172.21.15.40:0/2180632473 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f1d8c0fba00 0x7f1d8c0fddf0 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 rx=0 tx=0).stop 2023-11-17T05:36:35.420 INFO:teuthology.orchestra.run.smithi040.stderr:2023-11-17T05:36:35.415+0000 7f1d927ec700 1 --2- 172.21.15.40:0/2180632473 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f1d8c060310 0x7f1d8c0fb510 unknown :-1 s=CLOSED pgs=458 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:36:35.420 INFO:teuthology.orchestra.run.smithi040.stderr:2023-11-17T05:36:35.415+0000 7f1d927ec700 1 -- 172.21.15.40:0/2180632473 >> 172.21.15.40:0/2180632473 conn(0x7f1d8c0f0b90 msgr2=0x7f1d8c0f2fb0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:36:35.420 INFO:teuthology.orchestra.run.smithi040.stderr:2023-11-17T05:36:35.415+0000 7f1d927ec700 1 -- 172.21.15.40:0/2180632473 shutdown_connections 2023-11-17T05:36:35.420 INFO:teuthology.orchestra.run.smithi040.stderr:2023-11-17T05:36:35.415+0000 7f1d927ec700 1 -- 172.21.15.40:0/2180632473 wait complete. 2023-11-17T05:36:35.421 INFO:teuthology.orchestra.run.smithi040.stderr:2023-11-17T05:36:35.415+0000 7f1d927ec700 1 Processor -- start 2023-11-17T05:36:35.421 INFO:teuthology.orchestra.run.smithi040.stderr:2023-11-17T05:36:35.416+0000 7f1d927ec700 1 -- start start 2023-11-17T05:36:35.421 INFO:teuthology.orchestra.run.smithi040.stderr:2023-11-17T05:36:35.416+0000 7f1d927ec700 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f1d8c060310 0x7f1d8c108f00 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:36:35.421 INFO:teuthology.orchestra.run.smithi040.stderr:2023-11-17T05:36:35.416+0000 7f1d927ec700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f1d8c0fba00 0x7f1d8c1093f0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:36:35.421 INFO:teuthology.orchestra.run.smithi040.stderr:2023-11-17T05:36:35.416+0000 7f1d927ec700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f1d8c1099a0 con 0x7f1d8c0fba00 2023-11-17T05:36:35.421 INFO:teuthology.orchestra.run.smithi040.stderr:2023-11-17T05:36:35.416+0000 7f1d927ec700 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f1d8c109ae0 con 0x7f1d8c060310 2023-11-17T05:36:35.422 INFO:teuthology.orchestra.run.smithi040.stderr:2023-11-17T05:36:35.416+0000 7f1d90fe9700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f1d8c0fba00 0x7f1d8c1093f0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:36:35.422 INFO:teuthology.orchestra.run.smithi040.stderr:2023-11-17T05:36:35.416+0000 7f1d90fe9700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f1d8c0fba00 0x7f1d8c1093f0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:52698/0 (socket says 172.21.15.40:52698) 2023-11-17T05:36:35.422 INFO:teuthology.orchestra.run.smithi040.stderr:2023-11-17T05:36:35.416+0000 7f1d90fe9700 1 -- 172.21.15.40:0/2181923419 learned_addr learned my addr 172.21.15.40:0/2181923419 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-11-17T05:36:35.422 INFO:teuthology.orchestra.run.smithi040.stderr:2023-11-17T05:36:35.416+0000 7f1d917ea700 1 --2- 172.21.15.40:0/2181923419 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f1d8c060310 0x7f1d8c108f00 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:36:35.422 INFO:teuthology.orchestra.run.smithi040.stderr:2023-11-17T05:36:35.416+0000 7f1d90fe9700 1 -- 172.21.15.40:0/2181923419 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f1d8c060310 msgr2=0x7f1d8c108f00 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:36:35.422 INFO:teuthology.orchestra.run.smithi040.stderr:2023-11-17T05:36:35.416+0000 7f1d90fe9700 1 --2- 172.21.15.40:0/2181923419 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f1d8c060310 0x7f1d8c108f00 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:36:35.423 INFO:teuthology.orchestra.run.smithi040.stderr:2023-11-17T05:36:35.416+0000 7f1d90fe9700 1 -- 172.21.15.40:0/2181923419 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f1d88005040 con 0x7f1d8c0fba00 2023-11-17T05:36:35.423 INFO:teuthology.orchestra.run.smithi040.stderr:2023-11-17T05:36:35.416+0000 7f1d90fe9700 1 --2- 172.21.15.40:0/2181923419 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f1d8c0fba00 0x7f1d8c1093f0 secure :-1 s=READY pgs=459 cs=0 l=1 rev1=1 rx=0x7f1d7c00e7c0 tx=0x7f1d7c00b040).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:36:35.423 INFO:teuthology.orchestra.run.smithi040.stderr:2023-11-17T05:36:35.417+0000 7f1d81ffb700 1 -- 172.21.15.40:0/2181923419 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 357+0+0 (secure 0 0 0) 0x7f1d7c0092e0 con 0x7f1d8c0fba00 2023-11-17T05:36:35.423 INFO:teuthology.orchestra.run.smithi040.stderr:2023-11-17T05:36:35.417+0000 7f1d81ffb700 1 -- 172.21.15.40:0/2181923419 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(25 keys) v1 ==== 1074+0+0 (secure 0 0 0) 0x7f1d7c00f040 con 0x7f1d8c0fba00 2023-11-17T05:36:35.423 INFO:teuthology.orchestra.run.smithi040.stderr:2023-11-17T05:36:35.417+0000 7f1d927ec700 1 -- 172.21.15.40:0/2181923419 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f1d8c10a480 con 0x7f1d8c0fba00 2023-11-17T05:36:35.423 INFO:teuthology.orchestra.run.smithi040.stderr:2023-11-17T05:36:35.417+0000 7f1d81ffb700 1 -- 172.21.15.40:0/2181923419 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 357+0+0 (secure 0 0 0) 0x7f1d7c014730 con 0x7f1d8c0fba00 2023-11-17T05:36:35.424 INFO:teuthology.orchestra.run.smithi040.stderr:2023-11-17T05:36:35.417+0000 7f1d927ec700 1 -- 172.21.15.40:0/2181923419 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f1d8c10aa40 con 0x7f1d8c0fba00 2023-11-17T05:36:35.425 INFO:teuthology.orchestra.run.smithi040.stderr:2023-11-17T05:36:35.418+0000 7f1d81ffb700 1 -- 172.21.15.40:0/2181923419 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 18) v1 ==== 72193+0+0 (secure 0 0 0) 0x7f1d7c0077b0 con 0x7f1d8c0fba00 2023-11-17T05:36:35.426 INFO:teuthology.orchestra.run.smithi040.stderr:2023-11-17T05:36:35.418+0000 7f1d927ec700 1 -- 172.21.15.40:0/2181923419 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f1d8c045ad0 con 0x7f1d8c0fba00 2023-11-17T05:36:35.426 INFO:teuthology.orchestra.run.smithi040.stderr:2023-11-17T05:36:35.418+0000 7f1d81ffb700 1 --2- 172.21.15.40:0/2181923419 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f1d7805c2f0 0x7f1d7805e7b0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:36:35.426 INFO:teuthology.orchestra.run.smithi040.stderr:2023-11-17T05:36:35.419+0000 7f1d81ffb700 1 -- 172.21.15.40:0/2181923419 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(5..5 src has 1..5) v4 ==== 1270+0+0 (secure 0 0 0) 0x7f1d7c075270 con 0x7f1d8c0fba00 2023-11-17T05:36:35.426 INFO:teuthology.orchestra.run.smithi040.stderr:2023-11-17T05:36:35.419+0000 7f1d917ea700 1 --2- 172.21.15.40:0/2181923419 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f1d7805c2f0 0x7f1d7805e7b0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:36:35.426 INFO:teuthology.orchestra.run.smithi040.stderr:2023-11-17T05:36:35.419+0000 7f1d917ea700 1 --2- 172.21.15.40:0/2181923419 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f1d7805c2f0 0x7f1d7805e7b0 secure :-1 s=READY pgs=208 cs=0 l=1 rev1=1 rx=0x7f1d88004d80 tx=0x7f1d88016040).ready entity=mgr.14164 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:36:35.426 INFO:teuthology.orchestra.run.smithi040.stderr:2023-11-17T05:36:35.423+0000 7f1d81ffb700 1 -- 172.21.15.40:0/2181923419 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+152212 (secure 0 0 0) 0x7f1d7c048e80 con 0x7f1d8c0fba00 2023-11-17T05:36:35.620 INFO:teuthology.orchestra.run.smithi040.stderr:2023-11-17T05:36:35.618+0000 7f1d927ec700 1 -- 172.21.15.40:0/2181923419 --> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] -- mgr_command(tid 0: {"prefix": "orch daemon add osd", "svc_arg": "smithi040:vg_nvme/lv_4", "target": ["mon-mgr", ""]}) v1 -- 0x7f1d8c10ac60 con 0x7f1d7805c2f0 2023-11-17T05:36:35.988 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:36:35 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:36:35.568271+0000 mgr.smithi040.ogcprl (mgr.14164) 478 : cluster [DBG] pgmap v457: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:36:35.989 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:36:35 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:36:35.622080+0000 mon.smithi040 (mon.0) 384 : audit [DBG] from='mgr.14164 172.21.15.40:0/2203032451' entity='mgr.smithi040.ogcprl' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-11-17T05:36:35.989 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:36:35 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:36:35.623734+0000 mon.smithi040 (mon.0) 385 : audit [INF] from='mgr.14164 172.21.15.40:0/2203032451' entity='mgr.smithi040.ogcprl' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-11-17T05:36:35.989 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:36:35 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:36:35.624296+0000 mon.smithi040 (mon.0) 386 : audit [DBG] from='mgr.14164 172.21.15.40:0/2203032451' entity='mgr.smithi040.ogcprl' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-11-17T05:36:36.050 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:35 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: cluster 2023-11-17T05:36:35.568271+0000 mgr.smithi040.ogcprl (mgr.14164) 478 : cluster [DBG] pgmap v457: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:36:36.050 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:35 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: audit 2023-11-17T05:36:35.622080+0000 mon.smithi040 (mon.0) 384 : audit [DBG] from='mgr.14164 172.21.15.40:0/2203032451' entity='mgr.smithi040.ogcprl' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-11-17T05:36:36.050 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:35 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: audit 2023-11-17T05:36:35.623734+0000 mon.smithi040 (mon.0) 385 : audit [INF] from='mgr.14164 172.21.15.40:0/2203032451' entity='mgr.smithi040.ogcprl' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-11-17T05:36:36.050 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:35 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: audit 2023-11-17T05:36:35.624296+0000 mon.smithi040 (mon.0) 386 : audit [DBG] from='mgr.14164 172.21.15.40:0/2203032451' entity='mgr.smithi040.ogcprl' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-11-17T05:36:36.739 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:36:36 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:36:35.621121+0000 mgr.smithi040.ogcprl (mgr.14164) 479 : audit [DBG] from='client.14556 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi040:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2023-11-17T05:36:36.739 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:36:36 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cephadm 2023-11-17T05:36:35.623485+0000 mgr.smithi040.ogcprl (mgr.14164) 480 : cephadm [INF] Applying service osd.None on host smithi040... 2023-11-17T05:36:37.050 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:36 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: audit 2023-11-17T05:36:35.621121+0000 mgr.smithi040.ogcprl (mgr.14164) 479 : audit [DBG] from='client.14556 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi040:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2023-11-17T05:36:37.050 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:36 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: cephadm 2023-11-17T05:36:35.623485+0000 mgr.smithi040.ogcprl (mgr.14164) 480 : cephadm [INF] Applying service osd.None on host smithi040... 2023-11-17T05:36:37.989 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:36:37 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:36:37.568527+0000 mgr.smithi040.ogcprl (mgr.14164) 481 : cluster [DBG] pgmap v458: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:36:38.050 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:37 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: cluster 2023-11-17T05:36:37.568527+0000 mgr.smithi040.ogcprl (mgr.14164) 481 : cluster [DBG] pgmap v458: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:36:38.989 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:36:38 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:36:37.695742+0000 mon.smithi040 (mon.0) 387 : audit [INF] from='client.? 172.21.15.40:0/2654953923' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "6153a2e4-5c5c-4ea4-85b8-fbcee0397d79"}]: dispatch 2023-11-17T05:36:38.989 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:36:38 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:36:37.703634+0000 mon.smithi040 (mon.0) 388 : audit [INF] from='client.? 172.21.15.40:0/2654953923' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "6153a2e4-5c5c-4ea4-85b8-fbcee0397d79"}]': finished 2023-11-17T05:36:38.989 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:36:38 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:36:37.703704+0000 mon.smithi040 (mon.0) 389 : cluster [DBG] osdmap e6: 1 total, 0 up, 1 in 2023-11-17T05:36:38.989 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:36:38 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:36:37.703811+0000 mon.smithi040 (mon.0) 390 : audit [DBG] from='mgr.14164 172.21.15.40:0/2203032451' entity='mgr.smithi040.ogcprl' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-11-17T05:36:38.989 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:36:38 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:36:38.144358+0000 mon.smithi040 (mon.0) 391 : audit [DBG] from='client.? 172.21.15.40:0/799602753' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-11-17T05:36:39.050 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:38 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: audit 2023-11-17T05:36:37.695742+0000 mon.smithi040 (mon.0) 387 : audit [INF] from='client.? 172.21.15.40:0/2654953923' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "6153a2e4-5c5c-4ea4-85b8-fbcee0397d79"}]: dispatch 2023-11-17T05:36:39.050 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:38 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: audit 2023-11-17T05:36:37.703634+0000 mon.smithi040 (mon.0) 388 : audit [INF] from='client.? 172.21.15.40:0/2654953923' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "6153a2e4-5c5c-4ea4-85b8-fbcee0397d79"}]': finished 2023-11-17T05:36:39.050 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:38 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: cluster 2023-11-17T05:36:37.703704+0000 mon.smithi040 (mon.0) 389 : cluster [DBG] osdmap e6: 1 total, 0 up, 1 in 2023-11-17T05:36:39.050 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:38 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: audit 2023-11-17T05:36:37.703811+0000 mon.smithi040 (mon.0) 390 : audit [DBG] from='mgr.14164 172.21.15.40:0/2203032451' entity='mgr.smithi040.ogcprl' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-11-17T05:36:39.050 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:38 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: audit 2023-11-17T05:36:38.144358+0000 mon.smithi040 (mon.0) 391 : audit [DBG] from='client.? 172.21.15.40:0/799602753' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-11-17T05:36:40.252 INFO:teuthology.orchestra.run.smithi040.stderr:2023-11-17T05:36:40.250+0000 7f1d81ffb700 1 -- 172.21.15.40:0/2181923419 <== mgr.14164 v2:172.21.15.40:6800/1119389001 1 ==== mgr_command_reply(tid 0: -22 Traceback (most recent call last): 2023-11-17T05:36:40.252 INFO:teuthology.orchestra.run.smithi040.stderr: File "/usr/share/ceph/mgr/mgr_module.py", line 1335, in _handle_command 2023-11-17T05:36:40.252 INFO:teuthology.orchestra.run.smithi040.stderr: return self.handle_command(inbuf, cmd) 2023-11-17T05:36:40.252 INFO:teuthology.orchestra.run.smithi040.stderr: File "/usr/share/ceph/mgr/orchestrator/_interface.py", line 167, in handle_command 2023-11-17T05:36:40.253 INFO:teuthology.orchestra.run.smithi040.stderr: return dispatch[cmd['prefix']].call(self, cmd, inbuf) 2023-11-17T05:36:40.253 INFO:teuthology.orchestra.run.smithi040.stderr: File "/usr/share/ceph/mgr/mgr_module.py", line 389, in call 2023-11-17T05:36:40.253 INFO:teuthology.orchestra.run.smithi040.stderr: return self.func(mgr, **kwargs) 2023-11-17T05:36:40.253 INFO:teuthology.orchestra.run.smithi040.stderr: File "/usr/share/ceph/mgr/orchestrator/_interface.py", line 107, in 2023-11-17T05:36:40.253 INFO:teuthology.orchestra.run.smithi040.stderr: wrapper_copy = lambda *l_args, **l_kwargs: wrapper(*l_args, **l_kwargs) # noqa: E731 2023-11-17T05:36:40.253 INFO:teuthology.orchestra.run.smithi040.stderr: File "/usr/share/ceph/mgr/orchestrator/_interface.py", line 96, in wrapper 2023-11-17T05:36:40.253 INFO:teuthology.orchestra.run.smithi040.stderr: return func(*args, **kwargs) 2023-11-17T05:36:40.253 INFO:teuthology.orchestra.run.smithi040.stderr: File "/usr/share/ceph/mgr/orchestrator/module.py", line 795, in _daemon_add_osd 2023-11-17T05:36:40.253 INFO:teuthology.orchestra.run.smithi040.stderr: raise_if_exception(completion) 2023-11-17T05:36:40.253 INFO:teuthology.orchestra.run.smithi040.stderr: File "/usr/share/ceph/mgr/orchestrator/_interface.py", line 224, in raise_if_exception 2023-11-17T05:36:40.253 INFO:teuthology.orchestra.run.smithi040.stderr: raise e 2023-11-17T05:36:40.253 INFO:teuthology.orchestra.run.smithi040.stderr:RuntimeError: cephadm exited with an error code: 1, stderr:/bin/podman: --> passed data devices: 0 physical, 1 LVM 2023-11-17T05:36:40.253 INFO:teuthology.orchestra.run.smithi040.stderr:/bin/podman: --> relative data size: 1.0 2023-11-17T05:36:40.254 INFO:teuthology.orchestra.run.smithi040.stderr:/bin/podman: Running command: /usr/bin/ceph-authtool --gen-print-key 2023-11-17T05:36:40.254 INFO:teuthology.orchestra.run.smithi040.stderr:/bin/podman: Running command: /usr/bin/ceph --cluster ceph --name client.bootstrap-osd --keyring /var/lib/ceph/bootstrap-osd/ceph.keyring -i - osd new 6153a2e4-5c5c-4ea4-85b8-fbcee0397d79 2023-11-17T05:36:40.254 INFO:teuthology.orchestra.run.smithi040.stderr:/bin/podman: Running command: /usr/bin/ceph-authtool --gen-print-key 2023-11-17T05:36:40.254 INFO:teuthology.orchestra.run.smithi040.stderr:/bin/podman: Running command: /usr/bin/mount -t tmpfs tmpfs /var/lib/ceph/osd/ceph-0 2023-11-17T05:36:40.254 INFO:teuthology.orchestra.run.smithi040.stderr:/bin/podman: Running command: /usr/bin/chown -h ceph:ceph /dev/vg_nvme/lv_4 2023-11-17T05:36:40.254 INFO:teuthology.orchestra.run.smithi040.stderr:/bin/podman: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2023-11-17T05:36:40.254 INFO:teuthology.orchestra.run.smithi040.stderr:/bin/podman: Running command: /usr/bin/ln -s /dev/vg_nvme/lv_4 /var/lib/ceph/osd/ceph-0/block 2023-11-17T05:36:40.254 INFO:teuthology.orchestra.run.smithi040.stderr:/bin/podman: Running command: /usr/bin/ceph --cluster ceph --name client.bootstrap-osd --keyring /var/lib/ceph/bootstrap-osd/ceph.keyring mon getmap -o /var/lib/ceph/osd/ceph-0/activate.monmap 2023-11-17T05:36:40.254 INFO:teuthology.orchestra.run.smithi040.stderr:/bin/podman: stderr: 2023-11-17T05:36:37.904+0000 7f79ffbc0700 1 -- 172.21.15.40:0/1114973487 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f79f80ffa30 msgr2=0x7f79f8105c70 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:36:40.255 INFO:teuthology.orchestra.run.smithi040.stderr:/bin/podman: 2023-11-17T05:36:37.904+0000 7f79ffbc0700 1 --2- 172.21.15.40:0/1114973487 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f79f80ffa30 0x7f79f8105c70 secure :-1 s=READY pgs=462 cs=0 l=1 rev1=1 rx=0x7f79e80048c0 tx=0x7f79e8009b30).stop 2023-11-17T05:36:40.255 INFO:teuthology.orchestra.run.smithi040.stderr:/bin/podman: 2023-11-17T05:36:37.904+0000 7f79ffbc0700 1 -- 172.21.15.40:0/1114973487 shutdown_connections 2023-11-17T05:36:40.255 INFO:teuthology.orchestra.run.smithi040.stderr:/bin/podman: 2023-11-17T05:36:37.904+0000 7f79ffbc0700 1 --2- 172.21.15.40:0/1114973487 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f79f80ffa30 0x7f79f8105c70 unknown :-1 s=CLOSED pgs=462 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:36:40.255 INFO:teuthology.orchestra.run.smithi040.stderr:/bin/podman: 2023-11-17T05:36:37.904+0000 7f79ffbc0700 1 --2- 172.21.15.40:0/1114973487 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f79f80ff140 0x7f79f80ff540 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 rx=0 tx=0).stop 2023-11-17T05:36:40.255 INFO:teuthology.orchestra.run.smithi040.stderr:/bin/podman: 2023-11-17T05: 2023-11-17T05:36:40.255 INFO:teuthology.orchestra.run.smithi040.stderr:/bin/podman: stderr: 36:37.904+0000 7f79ffbc0700 1 -- 172.21.15.40:0/1114973487 >> 172.21.15.40:0/1114973487 conn(0x7f79f8065d40 msgr2=0x7f79f8068160 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:36:40.255 INFO:teuthology.orchestra.run.smithi040.stderr:/bin/podman: 2023-11-17T05:36:37.904+0000 7f79ffbc0700 1 -- 172.21.15.40:0/1114973487 shutdown_connections 2023-11-17T05:36:40.255 INFO:teuthology.orchestra.run.smithi040.stderr:/bin/podman: 2023-11-17T05:36:37.904+0000 7f79ffbc0700 1 -- 172.21.15.40:0/1114973487 wait complete. 2023-11-17T05:36:40.255 INFO:teuthology.orchestra.run.smithi040.stderr:/bin/podman: 2023-11-17T05:36:37.904+0000 7f79ffbc0700 1 Processor -- start 2023-11-17T05:36:40.255 INFO:teuthology.orchestra.run.smithi040.stderr:/bin/podman: stderr: 2023-11-17T05:36:37.904+0000 7f79ffbc0700 1 -- start start 2023-11-17T05:36:40.255 INFO:teuthology.orchestra.run.smithi040.stderr:/bin/podman: stderr: 2023-11-17T05:36:37.905+0000 7f79ffbc0700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f79f80ff140 0x7f79f8060ba0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:36:40.255 INFO:teuthology.orchestra.run.smithi040.stderr:/bin/podman: stderr: 2023-11-17T05:36:37.905+0000 7f79ffbc0700 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f79f80ffa30 0x7f79f8061090 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:36:40.255 INFO:teuthology.orchestra.run.smithi040.stderr:/bin/podman: 2023-11-17T05:36:37.905+0000 7f79ffbc0700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f79f80629f0 con 0x7f79f80ff140 2023-11-17T05:36:40.256 INFO:teuthology.orchestra.run.smithi040.stderr:/bin/podman: 2023-11-17T05:36:37.905+0000 7f79ffbc0700 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f79f8062b30 con 0x7f79f80ffa30 2023-11-17T05:36:40.256 INFO:teuthology.orchestra.run.smithi040.stderr:/bin/podman: 2023-11-17T05:36:37.905+0000 7f79fd95c700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f79f80ff140 0x7f79f8060ba0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:36:40.256 INFO:teuthology.orchestra.run.smithi040.stderr:/bin/podman: stderr: 2023-11-17T05:36:37.905+0000 7f79fd95c700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f79f80ff140 0x7f79f8060ba0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:52724/0 (socket says 172.21.15.40:52724) 2023-11-17T05:36:40.256 INFO:teuthology.orchestra.run.smithi040.stderr:/bin/podman: 2023-11-17T05:36:37.905+0000 7f79fd95c700 1 -- 172.21.15.40:0/799602753 learned_addr learned my addr 172.21.15.40:0/799602753 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-11-17T05:36:40.256 INFO:teuthology.orchestra.run.smithi040.stderr:/bin/podman: stderr: 2023-11-17T05:36:37.905+0000 7f79fd15b700 1 --2- 172.21.15.40:0/799602753 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f79f80ffa30 0x7f79f8061090 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:36:40.256 INFO:teuthology.orchestra.run.smithi040.stderr:/bin/podman: stderr: 2023-11-17T05:36:37.905+0000 7f79fd95c700 1 -- 172.21.15.40:0/799602753 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f79f80ffa30 msgr2=0x7f79f8061090 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:36:40.256 INFO:teuthology.orchestra.run.smithi040.stderr:/bin/podman: 2023-11-17T05:36:37.905+0000 7f79fd95c700 1 --2- 172.21.15.40:0/799602753 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f79f80ffa30 0x7f79f8061090 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:36:40.256 INFO:teuthology.orchestra.run.smithi040.stderr:/bin/podman: 2023-11-17T05:36:37.905+0000 7f79fd95c700 1 -- 172.21.15.40:0/799602753 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f79f8061580 con 0x7f79f80ff140 2023-11-17T05:36:40.256 INFO:teuthology.orchestra.run.smithi040.stderr:/bin/podman: stderr: 2023-11-17T05:36:37.905+0000 7f79fd95c700 1 --2- 172.21.15.40:0/799602753 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f79f80ff140 0x7f79f8060ba0 secure :-1 s=READY pgs=463 cs=0 l=1 rev1=1 rx=0x7f79f400e7c0 tx=0x7f79f400b040).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:36:40.256 INFO:teuthology.orchestra.run.smithi040.stderr:/bin/podman: stderr: 2023-11-17T05:36:37.905+0000 7f79ee7fc700 1 -- 172.21.15.40:0/799602753 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 357+0+0 (secure 0 0 0) 0x7f79f40092e0 con 0x7f79f80ff140 2023-11-17T05:36:40.256 INFO:teuthology.orchestra.run.smithi040.stderr:/bin/podman: 2023-11-17T05:36:37.905+0000 7f79ee7fc700 1 -- 172.21.15.40:0/799602753 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(25 keys) v1 ==== 1074+0+0 (secure 0 0 0) 0x7f79f400f040 con 0x7f79f80ff140 2023-11-17T05:36:40.256 INFO:teuthology.orchestra.run.smithi040.stderr:/bin/podman: stderr: 2023-11-17T05:36:37.905+0000 7f79ffbc0700 1 -- 172.21.15.40:0/799602753 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f79f8061830 con 0x7f79f80ff140 2023-11-17T05:36:40.256 INFO:teuthology.orchestra.run.smithi040.stderr:/bin/podman: 2023-11-17T05:36:37.905+0000 7f79ee7fc700 1 -- 172.21.15.40:0/799602753 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 357+0+0 (secure 0 0 0) 0x7f79f4014850 con 0x7f79f80ff140 2023-11-17T05:36:40.256 INFO:teuthology.orchestra.run.smithi040.stderr:/bin/podman: 2023-11-17T05:36:37.905+0000 7f79ffbc0700 1 -- 172.21.15.40:0/799602753 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f79f8118b50 con 0x7f79f80ff140 2023-11-17T05:36:40.257 INFO:teuthology.orchestra.run.smithi040.stderr:/bin/podman: stderr: 2023-11-17T05:36:37.907+0000 7f79ffbc0700 1 -- 172.21.15.40:0/799602753 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f79f8046570 con 0x7f79f80ff140 2023-11-17T05:36:40.257 INFO:teuthology.orchestra.run.smithi040.stderr:/bin/podman: stderr: 2023-11-17T05:36:37.907+0000 7f79ee7fc700 1 -- 172.21.15.40:0/799602753 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 18) v1 ==== 72193+0+0 (secure 0 0 0) 0x7f79f401b030 con 0x7f79f80ff140 2023-11-17T05:36:40.257 INFO:teuthology.orchestra.run.smithi040.stderr:/bin/podman: stderr: 2023-11-17T05:36:37.907+0000 7f79ee7fc700 1 --2- 172.21.15.40:0/799602753 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f79e405c370 0x7f79e405e830 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:36:40.257 INFO:teuthology.orchestra.run.smithi040.stderr:/bin/podman: stderr: 2023-11-17T05:36:37.907+0000 7f79ee7fc700 1 -- 172.21.15.40:0/799602753 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(6..6 src has 1..6) v4 ==== 1385+0+0 (secure 0 0 0) 0x7f79f4074e80 con 0x7f79f80ff140 2023-11-17T05:36:40.257 INFO:teuthology.orchestra.run.smithi040.stderr:/bin/podman: 2023-11-17T05:36:37.908+0000 7f79fd15b700 1 --2- 172.21.15.40:0/799602753 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f79e405c370 0x7f79e405e830 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:36:40.257 INFO:teuthology.orchestra.run.smithi040.stderr:/bin/podman: 2023-11-17T05:36:37.908+0000 7f79fd15b700 1 --2- 172.21.15.40:0/799602753 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f79e405c370 0x7f79e405e830 secure :-1 s=READY pgs=210 cs=0 l=1 rev1=1 rx=0x7f79e8004890 tx=0x7f79e800b040).ready entity=mgr.14164 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:36:40.257 INFO:teuthology.orchestra.run.smithi040.stderr:/bin/podman: stderr: 2023-11-17T05:36:37.911+0000 7f79ee7fc700 1 -- 172.21.15.40:0/799602753 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+152212 (secure 0 0 0) 0x7f79f4048ae0 con 0x7f79f80ff140 2023-11-17T05:36:40.257 INFO:teuthology.orchestra.run.smithi040.stderr:/bin/podman: stderr: 2023-11-17T05:36:38.142+0000 7f79ffbc0700 1 -- 172.21.15.40:0/799602753 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "mon getmap"} v 0) v1 -- 0x7f79f8108e40 con 0x7f79f80ff140 2023-11-17T05:36:40.257 INFO:teuthology.orchestra.run.smithi040.stderr:/bin/podman: stderr: 2023-11-17T05:36:38.142+0000 7f79ee7fc700 1 -- 172.21.15.40:0/799602753 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mon_command_ack([{"prefix": "mon getmap"}]=0 got monmap epoch 2 v2) v1 ==== 76+0+353 (secure 0 0 0) 0x7f79f4019030 con 0x7f79f80ff140 2023-11-17T05:36:40.257 INFO:teuthology.orchestra.run.smithi040.stderr:/bin/podman: stderr: 2023-11-17T05:36:38.145+0000 7f79ffbc0700 1 -- 172.21.15.40:0/799602753 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f79e405c370 msgr2=0x7f79e405e830 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:36:40.257 INFO:teuthology.orchestra.run.smithi040.stderr:/bin/podman: 2023-11-17T05:36:38.145+0000 7f79ffbc0700 1 --2- 172.21.15.40:0/799602753 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f79e405c370 0x7f79e405e830 secure :-1 s=READY pgs=210 cs=0 l=1 rev1=1 rx=0x7f79e8004890 tx=0x7f79e800b040).stop 2023-11-17T05:36:40.257 INFO:teuthology.orchestra.run.smithi040.stderr:/bin/podman: 2023-11-17T05:36:38.145+0000 7f79ffbc0700 1 -- 172.21.15.40:0/799602753 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f79f80ff140 msgr2=0x7f79f8060ba0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:36:40.258 INFO:teuthology.orchestra.run.smithi040.stderr:/bin/podman: 2023-11-17T05:36:38.145+0000 7f79ffbc0700 1 --2- 172.21.15.40:0/799602753 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f79f80ff140 0x7f79f8060ba0 secure :-1 s=READY pgs=463 cs=0 l=1 rev1=1 rx=0x7f79f400e7c0 tx=0x7f79f400b040).stop 2023-11-17T05:36:40.258 INFO:teuthology.orchestra.run.smithi040.stderr:/bin/podman: stderr: 2023-11-17T05:36:38.145+0000 7f79ffbc0700 1 -- 172.21.15.40:0/799602753 shutdown_connections 2023-11-17T05:36:40.258 INFO:teuthology.orchestra.run.smithi040.stderr:/bin/podman: 2023-11-17T05:36:38.145+0000 7f79ffbc0700 1 --2- 172.21.15.40:0/799602753 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f79e405c370 0x7f79e405e830 unknown :-1 s=CLOSED pgs=210 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:36:40.258 INFO:teuthology.orchestra.run.smithi040.stderr:/bin/podman: 2023-11-17T05:36:38.145+0000 7f79ffbc0700 1 --2- 172.21.15.40:0/799602753 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f79f80ff140 0x7f79f8060ba0 unknown :-1 s=CLOSED pgs=463 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:36:40.258 INFO:teuthology.orchestra.run.smithi040.stderr:/bin/podman: 2023-11-17T05:36:38.145+0000 7f79ffbc0700 1 --2- 172.21.15.40:0/799602753 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f79f80ffa30 0x7f79f8061090 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:36:40.268 INFO:teuthology.orchestra.run.smithi040.stderr:/bin/podman: 2023-11-17T05:36:38.145+0000 7f79ffbc0700 1 -- 172.21.15.40:0/799602753 >> 172.21.15.40:0/799602753 conn(0x7f79f8065d40 msgr2=0x7f79f8103470 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:36:40.269 INFO:teuthology.orchestra.run.smithi040.stderr:/bin/podman: 2023-11-17T05:36:38.145+0000 7f79ffbc0700 1 -- 172.21.15.40:0/79 2023-11-17T05:36:40.269 INFO:teuthology.orchestra.run.smithi040.stderr:/bin/podman: stderr: 9602753 shutdown_connections 2023-11-17T05:36:40.269 INFO:teuthology.orchestra.run.smithi040.stderr:/bin/podman: 2023-11-17T05:36:38.145+0000 7f79ffbc0700 1 -- 172.21.15.40:0/799602753 wait complete. 2023-11-17T05:36:40.269 INFO:teuthology.orchestra.run.smithi040.stderr:/bin/podman: stderr: got monmap epoch 2 2023-11-17T05:36:40.269 INFO:teuthology.orchestra.run.smithi040.stderr:/bin/podman: Running command: /usr/bin/ceph-authtool /var/lib/ceph/osd/ceph-0/keyring --create-keyring --name osd.0 --add-key AQDl+1ZlqWAuGBAALBRW+VnVLCAu4Sv5GMlYRg== 2023-11-17T05:36:40.269 INFO:teuthology.orchestra.run.smithi040.stderr:/bin/podman: stdout: creating /var/lib/ceph/osd/ceph-0/keyring 2023-11-17T05:36:40.269 INFO:teuthology.orchestra.run.smithi040.stderr:/bin/podman: added entity osd.0 auth(key=AQDl+1ZlqWAuGBAALBRW+VnVLCAu4Sv5GMlYRg==) 2023-11-17T05:36:40.269 INFO:teuthology.orchestra.run.smithi040.stderr:/bin/podman: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0/keyring 2023-11-17T05:36:40.269 INFO:teuthology.orchestra.run.smithi040.stderr:/bin/podman: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0/ 2023-11-17T05:36:40.269 INFO:teuthology.orchestra.run.smithi040.stderr:/bin/podman: Running command: /usr/bin/ceph-osd --cluster ceph --osd-objectstore bluestore --mkfs -i 0 --monmap /var/lib/ceph/osd/ceph-0/activate.monmap --keyfile - --osdspec-affinity None --osd-data /var/lib/ceph/osd/ceph-0/ --osd-uuid 6153a2e4-5c5c-4ea4-85b8-fbcee0397d79 --setuser ceph --setgroup ceph 2023-11-17T05:36:40.269 INFO:teuthology.orchestra.run.smithi040.stderr:/bin/podman: stderr: 2023-11-17T05:36:38.203+0000 7f854fc87080 -1 bluestore(/var/lib/ceph/osd/ceph-0/) _read_fsid unparsable uuid 2023-11-17T05:36:40.269 INFO:teuthology.orchestra.run.smithi040.stderr:/bin/podman: stderr: 2023-11-17T05:36:38.563+0000 7f854fc87080 -1 bluefs _replay 0x0: stop: uuid 00000000-0000-0000-0000-000000000000 != super.uuid 0ffb7a6f-1ef2-4d96-af47-8d61bbdc0809, block dump: 2023-11-17T05:36:40.269 INFO:teuthology.orchestra.run.smithi040.stderr:/bin/podman: stderr: 00000000 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................| 2023-11-17T05:36:40.270 INFO:teuthology.orchestra.run.smithi040.stderr:/bin/podman: stderr: * 2023-11-17T05:36:40.270 INFO:teuthology.orchestra.run.smithi040.stderr:/bin/podman: stderr: 00000ff0 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................| 2023-11-17T05:36:40.270 INFO:teuthology.orchestra.run.smithi040.stderr:/bin/podman: stderr: 00001000 2023-11-17T05:36:40.270 INFO:teuthology.orchestra.run.smithi040.stderr:/bin/podman: stderr: 2023-11-17T05:36:38.974+0000 7f854fc87080 -1 bluestore(/var/lib/ceph/osd/ceph-0/) _open_db erroring opening db: 2023-11-17T05:36:40.270 INFO:teuthology.orchestra.run.smithi040.stderr:/bin/podman: stderr: 2023-11-17T05:36:39.486+0000 7f854fc87080 -1 OSD::mkfs: ObjectStore::mkfs failed with error (5) Input/output error 2023-11-17T05:36:40.270 INFO:teuthology.orchestra.run.smithi040.stderr:/bin/podman: stderr: 2023-11-17T05:36:39.486+0000 7f854fc87080 -1  ** ERROR: error creating empty object store in /var/lib/ceph/osd/ceph-0/: (5) Input/output error 2023-11-17T05:36:40.270 INFO:teuthology.orchestra.run.smithi040.stderr:/bin/podman: --> Was unable to complete a new OSD, will rollback changes 2023-11-17T05:36:40.270 INFO:teuthology.orchestra.run.smithi040.stderr:/bin/podman: Running command: /usr/bin/ceph --cluster ceph --name client.bootstrap-osd --keyring /var/lib/ceph/bootstrap-osd/ceph.keyring osd purge-new osd.0 --yes-i-really-mean-it 2023-11-17T05:36:40.270 INFO:teuthology.orchestra.run.smithi040.stderr:/bin/podman: stderr: 2023-11-17T05:36:39.601+0000 7f9603047700 1 -- 172.21.15.40:0/3178258714 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f95fc0ffa10 msgr2=0x7f95fc105c50 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:36:40.270 INFO:teuthology.orchestra.run.smithi040.stderr:/bin/podman: 2023-11-17T05:36:39.601+0000 7f9603047700 1 --2- 172.21.15.40:0/3178258714 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f95fc0ffa10 0x7f95fc105c50 secure :-1 s=READY pgs=465 cs=0 l=1 rev1=1 rx=0x7f95e80048f0 tx=0x7f95e8009b30).stop 2023-11-17T05:36:40.270 INFO:teuthology.orchestra.run.smithi040.stderr:/bin/podman: 2023-11-17T05:36:39.601+0000 7f9603047700 1 -- 172.21.15.40:0/3178258714 shutdown_connections 2023-11-17T05:36:40.270 INFO:teuthology.orchestra.run.smithi040.stderr:/bin/podman: 2023-11-17T05:36:39.601+0000 7f9603047700 1 --2- 172.21.15.40:0/3178258714 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f95fc0ffa10 0x7f95fc105c50 unknown :-1 s=CLOSED pgs=465 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:36:40.270 INFO:teuthology.orchestra.run.smithi040.stderr:/bin/podman: 2023-11-17T05:36:39.601+0000 7f9603047700 1 --2- 172.21.15.40:0/3178258714 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f95fc0ff120 0x7f95fc0ff520 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 rx=0 tx=0).stop 2023-11-17T05:36:40.271 INFO:teuthology.orchestra.run.smithi040.stderr:/bin/podman: 2023-11-17T05: 2023-11-17T05:36:40.271 INFO:teuthology.orchestra.run.smithi040.stderr:/bin/podman: stderr: 36:39.601+0000 7f9603047700 1 -- 172.21.15.40:0/3178258714 >> 172.21.15.40:0/3178258714 conn(0x7f95fc065d90 msgr2=0x7f95fc0681b0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:36:40.271 INFO:teuthology.orchestra.run.smithi040.stderr:/bin/podman: 2023-11-17T05:36:39.601+0000 7f9603047700 1 -- 172.21.15.40:0/3178258714 shutdown_connections 2023-11-17T05:36:40.271 INFO:teuthology.orchestra.run.smithi040.stderr:/bin/podman: 2023-11-17T05:36:39.601+0000 7f9603047700 1 -- 172.21.15.40:0/3178258714 wait complete. 2023-11-17T05:36:40.271 INFO:teuthology.orchestra.run.smithi040.stderr:/bin/podman: stderr: 2023-11-17T05:36:39.601+0000 7f9603047700 1 Processor -- start 2023-11-17T05:36:40.271 INFO:teuthology.orchestra.run.smithi040.stderr:/bin/podman: stderr: 2023-11-17T05:36:39.601+0000 7f9603047700 1 -- start start 2023-11-17T05:36:40.271 INFO:teuthology.orchestra.run.smithi040.stderr:/bin/podman: stderr: 2023-11-17T05:36:39.602+0000 7f9603047700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f95fc0ff120 0x7f95fc112320 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:36:40.271 INFO:teuthology.orchestra.run.smithi040.stderr:/bin/podman: 2023-11-17T05:36:39.602+0000 7f9603047700 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f95fc0ffa10 0x7f95fc112810 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:36:40.271 INFO:teuthology.orchestra.run.smithi040.stderr:/bin/podman: stderr: 2023-11-17T05:36:39.602+0000 7f9603047700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f95fc112e10 con 0x7f95fc0ff120 2023-11-17T05:36:40.271 INFO:teuthology.orchestra.run.smithi040.stderr:/bin/podman: 2023-11-17T05:36:39.602+0000 7f9603047700 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f95fc10f580 con 0x7f95fc0ffa10 2023-11-17T05:36:40.271 INFO:teuthology.orchestra.run.smithi040.stderr:/bin/podman: stderr: 2023-11-17T05:36:39.602+0000 7f9600de3700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f95fc0ff120 0x7f95fc112320 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:36:40.271 INFO:teuthology.orchestra.run.smithi040.stderr:/bin/podman: stderr: 2023-11-17T05:36:39.602+0000 7f9600de3700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f95fc0ff120 0x7f95fc112320 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:52746/0 (socket says 172.21.15.40:52746) 2023-11-17T05:36:40.271 INFO:teuthology.orchestra.run.smithi040.stderr:/bin/podman: 2023-11-17T05:36:39.602+0000 7f9600de3700 1 -- 172.21.15.40:0/1216118669 learned_addr learned my addr 172.21.15.40:0/1216118669 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-11-17T05:36:40.271 INFO:teuthology.orchestra.run.smithi040.stderr:/bin/podman: stderr: 2023-11-17T05:36:39.602+0000 7f95fbfff700 1 --2- 172.21.15.40:0/1216118669 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f95fc0ffa10 0x7f95fc112810 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:36:40.272 INFO:teuthology.orchestra.run.smithi040.stderr:/bin/podman: stderr: 2023-11-17T05:36:39.602+0000 7f9600de3700 1 -- 172.21.15.40:0/1216118669 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f95fc0ffa10 msgr2=0x7f95fc112810 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:36:40.272 INFO:teuthology.orchestra.run.smithi040.stderr:/bin/podman: 2023-11-17T05:36:39.602+0000 7f9600de3700 1 --2- 172.21.15.40:0/1216118669 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f95fc0ffa10 0x7f95fc112810 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:36:40.272 INFO:teuthology.orchestra.run.smithi040.stderr:/bin/podman: 2023-11-17T05:36:39.602+0000 7f9600de3700 1 -- 172.21.15.40:0/1216118669 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f95fc10f740 con 0x7f95fc0ff120 2023-11-17T05:36:40.272 INFO:teuthology.orchestra.run.smithi040.stderr:/bin/podman: 2023-11-17T05:36:39.602+0000 7f95fbfff700 1 --2- 172.21.15.40:0/1216118669 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f95fc0ffa10 0x7f95fc112810 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 rx=0 tx=0).send_auth_request state changed! 2023-11-17T05:36:40.272 INFO:teuthology.orchestra.run.smithi040.stderr:/bin/podman: stderr: 2023-11-17T05:36:39.602+0000 7f9600de3700 1 --2- 172.21.15.40:0/1216118669 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f95fc0ff120 0x7f95fc112320 secure :-1 s=READY pgs=466 cs=0 l=1 rev1=1 rx=0x7f95f000c580 tx=0x7f95f0007e40).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:36:40.272 INFO:teuthology.orchestra.run.smithi040.stderr:/bin/podman: stderr: 2023-11-17T05:36:39.603+0000 7f95f97fa700 1 -- 172.21.15.40:0/1216118669 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 357+0+0 (secure 0 0 0) 0x7f95f000e680 con 0x7f95fc0ff120 2023-11-17T05:36:40.272 INFO:teuthology.orchestra.run.smithi040.stderr:/bin/podman: stderr: 2023-11-17T05:36:39.603+0000 7f9603047700 1 -- 172.21.15.40:0/1216118669 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f95fc10f9f0 con 0x7f95fc0ff120 2023-11-17T05:36:40.272 INFO:teuthology.orchestra.run.smithi040.stderr:/bin/podman: 2023-11-17T05:36:39.603+0000 7f95f97fa700 1 -- 172.21.15.40:0/1216118669 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(25 keys) v1 ==== 1074+0+0 (secure 0 0 0) 0x7f95f000ec90 con 0x7f95fc0ff120 2023-11-17T05:36:40.272 INFO:teuthology.orchestra.run.smithi040.stderr:/bin/podman: 2023-11-17T05:36:39.603+0000 7f95f97fa700 1 -- 172.21.15.40:0/1216118669 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 357+0+0 (secure 0 0 0) 0x7f95f0012ca0 con 0x7f95fc0ff120 2023-11-17T05:36:40.272 INFO:teuthology.orchestra.run.smithi040.stderr:/bin/podman: stderr: 2023-11-17T05:36:39.603+0000 7f9603047700 1 -- 172.21.15.40:0/1216118669 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f95fc10ff30 con 0x7f95fc0ff120 2023-11-17T05:36:40.272 INFO:teuthology.orchestra.run.smithi040.stderr:/bin/podman: stderr: 2023-11-17T05:36:39.604+0000 7f9603047700 1 -- 172.21.15.40:0/1216118669 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f95fc001550 con 0x7f95fc0ff120 2023-11-17T05:36:40.272 INFO:teuthology.orchestra.run.smithi040.stderr:/bin/podman: stderr: 2023-11-17T05:36:39.604+0000 7f95f97fa700 1 -- 172.21.15.40:0/1216118669 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 18) v1 ==== 72193+0+0 (secure 0 0 0) 0x7f95f0019030 con 0x7f95fc0ff120 2023-11-17T05:36:40.272 INFO:teuthology.orchestra.run.smithi040.stderr:/bin/podman: stderr: 2023-11-17T05:36:39.605+0000 7f95f97fa700 1 --2- 172.21.15.40:0/1216118669 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f95ec05c360 0x7f95ec05e820 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:36:40.273 INFO:teuthology.orchestra.run.smithi040.stderr:/bin/podman: stderr: 2023-11-17T05:36:39.605+0000 7f95f97fa700 1 -- 172.21.15.40:0/1216118669 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(6..6 src has 1..6) v4 ==== 1385+0+0 (secure 0 0 0) 0x7f95f0074c90 con 0x7f95fc0ff120 2023-11-17T05:36:40.273 INFO:teuthology.orchestra.run.smithi040.stderr:/bin/podman: 2023-11-17T05:36:39.605+0000 7f95fbfff700 1 --2- 172.21.15.40:0/1216118669 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f95ec05c360 0x7f95ec05e820 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:36:40.273 INFO:teuthology.orchestra.run.smithi040.stderr:/bin/podman: stderr: 2023-11-17T05:36:39.605+0000 7f95fbfff700 1 --2- 172.21.15.40:0/1216118669 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f95ec05c360 0x7f95ec05e820 secure :-1 s=READY pgs=211 cs=0 l=1 rev1=1 rx=0x7f95e800b370 tx=0x7f95e8009b30).ready entity=mgr.14164 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:36:40.273 INFO:teuthology.orchestra.run.smithi040.stderr:/bin/podman: stderr: 2023-11-17T05:36:39.609+0000 7f95f97fa700 1 -- 172.21.15.40:0/1216118669 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+152212 (secure 0 0 0) 0x7f95f00462b0 con 0x7f95fc0ff120 2023-11-17T05:36:40.273 INFO:teuthology.orchestra.run.smithi040.stderr:/bin/podman: stderr: 2023-11-17T05:36:39.787+0000 7f9603047700 1 -- 172.21.15.40:0/1216118669 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "osd purge-new", "id": 0, "yes_i_really_mean_it": true} v 0) v1 -- 0x7f95fc108e00 con 0x7f95fc0ff120 2023-11-17T05:36:40.273 INFO:teuthology.orchestra.run.smithi040.stderr:/bin/podman: stderr: 2023-11-17T05:36:39.792+0000 7f95f97fa700 1 -- 172.21.15.40:0/1216118669 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mon_command_ack([{"prefix": "osd purge-new", "id": 0, "yes_i_really_mean_it": true}]=0 purged osd.0 v7) v1 ==== 112+0+0 (secure 0 0 0) 0x7f95f0017030 con 0x7f95fc0ff120 2023-11-17T05:36:40.273 INFO:teuthology.orchestra.run.smithi040.stderr:/bin/podman: stderr: 2023-11-17T05:36:39.793+0000 7f9603047700 1 -- 172.21.15.40:0/1216118669 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f95ec05c360 msgr2=0x7f95ec05e820 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:36:40.273 INFO:teuthology.orchestra.run.smithi040.stderr:/bin/podman: 2023-11-17T05:36:39.793+0000 7f9603047700 1 --2- 172.21.15.40:0/1216118669 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f95ec05c360 0x7f95ec05e820 secure :-1 s=READY pgs=211 cs=0 l=1 rev1=1 rx=0x7f95e800b370 tx=0x7f95e8009b30).stop 2023-11-17T05:36:40.273 INFO:teuthology.orchestra.run.smithi040.stderr:/bin/podman: 2023-11-17T05:36:39.793+0000 7f9603047700 1 -- 172.21.15.40:0/1216118669 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f95fc0ff120 msgr2=0x7f95fc112320 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:36:40.273 INFO:teuthology.orchestra.run.smithi040.stderr:/bin/podman: 2023-11-17T05:36:39.793+0000 7f9603047700 1 --2- 172.21.15.40:0/1216118669 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f95fc0ff120 0x7f95fc112320 secure :-1 s=READY pgs=466 cs=0 l=1 rev1=1 rx=0x7f95f000c580 tx=0x7f95f0007e40).stop 2023-11-17T05:36:40.273 INFO:teuthology.orchestra.run.smithi040.stderr:/bin/podman: stderr: 2023-11-17T05:36:39.794+0000 7f9603047700 1 -- 172.21.15.40:0/1216118669 shutdown_connections 2023-11-17T05:36:40.273 INFO:teuthology.orchestra.run.smithi040.stderr:/bin/podman: 2023-11-17T05:36:39.794+0000 7f9603047700 1 --2- 172.21.15.40:0/1216118669 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f95ec05c360 0x7f95ec05e820 unknown :-1 s=CLOSED pgs=211 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:36:40.273 INFO:teuthology.orchestra.run.smithi040.stderr:/bin/podman: 2023-11-17T05:36:39.794+0000 7f9603047700 1 --2- 172.21.15.40:0/1216118669 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f95fc0ff120 0x7f95fc112320 unknown :-1 s=CLOSED pgs=466 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:36:40.274 INFO:teuthology.orchestra.run.smithi040.stderr:/bin/podman: 2023-11-17T05:36:39.794+0000 7f9603047700 1 --2- 172.21.15.40:0/1216118669 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f95fc0ffa10 0x7f95fc112810 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:36:40.274 INFO:teuthology.orchestra.run.smithi040.stderr:/bin/podman: 2023-11-17T05:36:39.794+0000 7f9603047700 1 -- 172.21.15.40:0/1216118669 >> 172.21.15.40:0/1216118669 conn(0x7f95fc065d90 msgr2=0x7f95fc103450 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:36:40.274 INFO:teuthology.orchestra.run.smithi040.stderr:/bin/podman: 2023-11-17T05:36:39.794+0000 7f9603047700 1 -- 172.21.15.4 2023-11-17T05:36:40.274 INFO:teuthology.orchestra.run.smithi040.stderr:/bin/podman: stderr: 0:0/1216118669 shutdown_connections 2023-11-17T05:36:40.274 INFO:teuthology.orchestra.run.smithi040.stderr:/bin/podman: 2023-11-17T05:36:39.794+0000 7f9603047700 1 -- 172.21.15.40:0/1216118669 wait complete. 2023-11-17T05:36:40.274 INFO:teuthology.orchestra.run.smithi040.stderr:/bin/podman: stderr: purged osd.0 2023-11-17T05:36:40.274 INFO:teuthology.orchestra.run.smithi040.stderr:/bin/podman: --> RuntimeError: Command failed with exit code 250: /usr/bin/ceph-osd --cluster ceph --osd-objectstore bluestore --mkfs -i 0 --monmap /var/lib/ceph/osd/ceph-0/activate.monmap --keyfile - --osdspec-affinity None --osd-data /var/lib/ceph/osd/ceph-0/ --osd-uuid 6153a2e4-5c5c-4ea4-85b8-fbcee0397d79 --setuser ceph --setgroup ceph 2023-11-17T05:36:40.274 INFO:teuthology.orchestra.run.smithi040.stderr:Traceback (most recent call last): 2023-11-17T05:36:40.274 INFO:teuthology.orchestra.run.smithi040.stderr: File "/var/lib/ceph/ed6470e2-8508-11ee-8dbe-212e2dc638e7/cephadm.30cb78bdbbafb384af862e1c2292b944f15942b586128e91262b43e91e11ae90", line 8029, in 2023-11-17T05:36:40.274 INFO:teuthology.orchestra.run.smithi040.stderr: main() 2023-11-17T05:36:40.274 INFO:teuthology.orchestra.run.smithi040.stderr: File "/var/lib/ceph/ed6470e2-8508-11ee-8dbe-212e2dc638e7/cephadm.30cb78bdbbafb384af862e1c2292b944f15942b586128e91262b43e91e11ae90", line 8017, in main 2023-11-17T05:36:40.274 INFO:teuthology.orchestra.run.smithi040.stderr: r = ctx.func(ctx) 2023-11-17T05:36:40.274 INFO:teuthology.orchestra.run.smithi040.stderr: File "/var/lib/ceph/ed6470e2-8508-11ee-8dbe-212e2dc638e7/cephadm.30cb78bdbbafb384af862e1c2292b944f15942b586128e91262b43e91e11ae90", line 1654, in _infer_fsid 2023-11-17T05:36:40.275 INFO:teuthology.orchestra.run.smithi040.stderr: return func(ctx) 2023-11-17T05:36:40.275 INFO:teuthology.orchestra.run.smithi040.stderr: File "/var/lib/ceph/ed6470e2-8508-11ee-8dbe-212e2dc638e7/cephadm.30cb78bdbbafb384af862e1c2292b944f15942b586128e91262b43e91e11ae90", line 1738, in _infer_image 2023-11-17T05:36:40.275 INFO:teuthology.orchestra.run.smithi040.stderr: return func(ctx) 2023-11-17T05:36:40.275 INFO:teuthology.orchestra.run.smithi040.stderr: File "/var/lib/ceph/ed6470e2-8508-11ee-8dbe-212e2dc638e7/cephadm.30cb78bdbbafb384af862e1c2292b944f15942b586128e91262b43e91e11ae90", line 4514, in command_ceph_volume 2023-11-17T05:36:40.275 INFO:teuthology.orchestra.run.smithi040.stderr: out, err, code = call_throws(ctx, c.run_cmd(), verbosity=verbosity) 2023-11-17T05:36:40.275 INFO:teuthology.orchestra.run.smithi040.stderr: File "/var/lib/ceph/ed6470e2-8508-11ee-8dbe-212e2dc638e7/cephadm.30cb78bdbbafb384af862e1c2292b944f15942b586128e91262b43e91e11ae90", line 1464, in call_throws 2023-11-17T05:36:40.275 INFO:teuthology.orchestra.run.smithi040.stderr: raise RuntimeError('Failed command: %s' % ' '.join(command)) 2023-11-17T05:36:40.275 INFO:teuthology.orchestra.run.smithi040.stderr:RuntimeError: Failed command: /bin/podman run --rm --ipc=host --no-hosts --net=host --entrypoint /usr/sbin/ceph-volume --privileged --group-add=disk --init -e CONTAINER_IMAGE=docker.io/ceph/ceph@sha256:54e95ae1e11404157d7b329d0bef866ebbb214b195a009e87aae4eba9d282949 -e NODE_NAME=smithi040 -e CEPH_USE_RANDOM_NONCE=1 -e CEPH_VOLUME_OSDSPEC_AFFINITY=None -v /var/run/ceph/ed6470e2-8508-11ee-8dbe-212e2dc638e7:/var/run/ceph:z -v /var/log/ceph/ed6470e2-8508-11ee-8dbe-212e2dc638e7:/var/log/ceph:z -v /var/lib/ceph/ed6470e2-8508-11ee-8dbe-212e2dc638e7/crash:/var/lib/ceph/crash:z -v /dev:/dev -v /run/udev:/run/udev -v /sys:/sys -v /var/lib/ceph/ed6470e2-8508-11ee-8dbe-212e2dc638e7/selinux:/sys/fs/selinux:ro -v /run/lvm:/run/lvm -v /run/lock/lvm:/run/lock/lvm -v /tmp/ceph-tmpz30t5yd6:/etc/ceph/ceph.conf:z -v /tmp/ceph-tmpgoj2newa:/var/lib/ceph/bootstrap-osd/ceph.keyring:z docker.io/ceph/ceph@sha256:54e95ae1e11404157d7b329d0bef866ebbb214b195a009e87aae4eba9d282949 lvm batch --no-auto vg_nvme/lv_4 --yes --no-systemd 2023-11-17T05:36:40.275 INFO:teuthology.orchestra.run.smithi040.stderr:) v1 ==== 27637+0+0 (secure 0 0 0) 0x7f1d8c10ac60 con 0x7f1d7805c2f0 2023-11-17T05:36:40.275 INFO:teuthology.orchestra.run.smithi040.stderr:Error EINVAL: Traceback (most recent call last): 2023-11-17T05:36:40.275 INFO:teuthology.orchestra.run.smithi040.stderr: File "/usr/share/ceph/mgr/mgr_module.py", line 1335, in _handle_command 2023-11-17T05:36:40.275 INFO:teuthology.orchestra.run.smithi040.stderr: return self.handle_command(inbuf, cmd) 2023-11-17T05:36:40.275 INFO:teuthology.orchestra.run.smithi040.stderr: File "/usr/share/ceph/mgr/orchestrator/_interface.py", line 167, in handle_command 2023-11-17T05:36:40.275 INFO:teuthology.orchestra.run.smithi040.stderr: return dispatch[cmd['prefix']].call(self, cmd, inbuf) 2023-11-17T05:36:40.276 INFO:teuthology.orchestra.run.smithi040.stderr: File "/usr/share/ceph/mgr/mgr_module.py", line 389, in call 2023-11-17T05:36:40.276 INFO:teuthology.orchestra.run.smithi040.stderr: return self.func(mgr, **kwargs) 2023-11-17T05:36:40.276 INFO:teuthology.orchestra.run.smithi040.stderr: File "/usr/share/ceph/mgr/orchestrator/_interface.py", line 107, in 2023-11-17T05:36:40.276 INFO:teuthology.orchestra.run.smithi040.stderr: wrapper_copy = lambda *l_args, **l_kwargs: wrapper(*l_args, **l_kwargs) # noqa: E731 2023-11-17T05:36:40.276 INFO:teuthology.orchestra.run.smithi040.stderr: File "/usr/share/ceph/mgr/orchestrator/_interface.py", line 96, in wrapper 2023-11-17T05:36:40.276 INFO:teuthology.orchestra.run.smithi040.stderr: return func(*args, **kwargs) 2023-11-17T05:36:40.276 INFO:teuthology.orchestra.run.smithi040.stderr: File "/usr/share/ceph/mgr/orchestrator/module.py", line 795, in _daemon_add_osd 2023-11-17T05:36:40.276 INFO:teuthology.orchestra.run.smithi040.stderr: raise_if_exception(completion) 2023-11-17T05:36:40.276 INFO:teuthology.orchestra.run.smithi040.stderr: File "/usr/share/ceph/mgr/orchestrator/_interface.py", line 224, in raise_if_exception 2023-11-17T05:36:40.276 INFO:teuthology.orchestra.run.smithi040.stderr: raise e 2023-11-17T05:36:40.276 INFO:teuthology.orchestra.run.smithi040.stderr:RuntimeError: cephadm exited with an error code: 1, stderr:/bin/podman: --> passed data devices: 0 physical, 1 LVM 2023-11-17T05:36:40.276 INFO:teuthology.orchestra.run.smithi040.stderr:/bin/podman: --> relative data size: 1.0 2023-11-17T05:36:40.276 INFO:teuthology.orchestra.run.smithi040.stderr:/bin/podman: Running command: /usr/bin/ceph-authtool --gen-print-key 2023-11-17T05:36:40.276 INFO:teuthology.orchestra.run.smithi040.stderr:/bin/podman: Running command: /usr/bin/ceph --cluster ceph --name client.bootstrap-osd --keyring /var/lib/ceph/bootstrap-osd/ceph.keyring -i - osd new 6153a2e4-5c5c-4ea4-85b8-fbcee0397d79 2023-11-17T05:36:40.277 INFO:teuthology.orchestra.run.smithi040.stderr:/bin/podman: Running command: /usr/bin/ceph-authtool --gen-print-key 2023-11-17T05:36:40.277 INFO:teuthology.orchestra.run.smithi040.stderr:/bin/podman: Running command: /usr/bin/mount -t tmpfs tmpfs /var/lib/ceph/osd/ceph-0 2023-11-17T05:36:40.277 INFO:teuthology.orchestra.run.smithi040.stderr:/bin/podman: Running command: /usr/bin/chown -h ceph:ceph /dev/vg_nvme/lv_4 2023-11-17T05:36:40.277 INFO:teuthology.orchestra.run.smithi040.stderr:/bin/podman: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2023-11-17T05:36:40.277 INFO:teuthology.orchestra.run.smithi040.stderr:/bin/podman: Running command: /usr/bin/ln -s /dev/vg_nvme/lv_4 /var/lib/ceph/osd/ceph-0/block 2023-11-17T05:36:40.277 INFO:teuthology.orchestra.run.smithi040.stderr:/bin/podman: Running command: /usr/bin/ceph --cluster ceph --name client.bootstrap-osd --keyring /var/lib/ceph/bootstrap-osd/ceph.keyring mon getmap -o /var/lib/ceph/osd/ceph-0/activate.monmap 2023-11-17T05:36:40.277 INFO:teuthology.orchestra.run.smithi040.stderr:/bin/podman: stderr: 2023-11-17T05:36:37.904+0000 7f79ffbc0700 1 -- 172.21.15.40:0/1114973487 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f79f80ffa30 msgr2=0x7f79f8105c70 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:36:40.277 INFO:teuthology.orchestra.run.smithi040.stderr:/bin/podman: 2023-11-17T05:36:37.904+0000 7f79ffbc0700 1 --2- 172.21.15.40:0/1114973487 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f79f80ffa30 0x7f79f8105c70 secure :-1 s=READY pgs=462 cs=0 l=1 rev1=1 rx=0x7f79e80048c0 tx=0x7f79e8009b30).stop 2023-11-17T05:36:40.277 INFO:teuthology.orchestra.run.smithi040.stderr:/bin/podman: 2023-11-17T05:36:37.904+0000 7f79ffbc0700 1 -- 172.21.15.40:0/1114973487 shutdown_connections 2023-11-17T05:36:40.277 INFO:teuthology.orchestra.run.smithi040.stderr:/bin/podman: 2023-11-17T05:36:37.904+0000 7f79ffbc0700 1 --2- 172.21.15.40:0/1114973487 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f79f80ffa30 0x7f79f8105c70 unknown :-1 s=CLOSED pgs=462 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:36:40.277 INFO:teuthology.orchestra.run.smithi040.stderr:/bin/podman: 2023-11-17T05:36:37.904+0000 7f79ffbc0700 1 --2- 172.21.15.40:0/1114973487 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f79f80ff140 0x7f79f80ff540 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 rx=0 tx=0).stop 2023-11-17T05:36:40.277 INFO:teuthology.orchestra.run.smithi040.stderr:/bin/podman: 2023-11-17T05: 2023-11-17T05:36:40.277 INFO:teuthology.orchestra.run.smithi040.stderr:/bin/podman: stderr: 36:37.904+0000 7f79ffbc0700 1 -- 172.21.15.40:0/1114973487 >> 172.21.15.40:0/1114973487 conn(0x7f79f8065d40 msgr2=0x7f79f8068160 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:36:40.277 INFO:teuthology.orchestra.run.smithi040.stderr:/bin/podman: 2023-11-17T05:36:37.904+0000 7f79ffbc0700 1 -- 172.21.15.40:0/1114973487 shutdown_connections 2023-11-17T05:36:40.278 INFO:teuthology.orchestra.run.smithi040.stderr:/bin/podman: 2023-11-17T05:36:37.904+0000 7f79ffbc0700 1 -- 172.21.15.40:0/1114973487 wait complete. 2023-11-17T05:36:40.278 INFO:teuthology.orchestra.run.smithi040.stderr:/bin/podman: 2023-11-17T05:36:37.904+0000 7f79ffbc0700 1 Processor -- start 2023-11-17T05:36:40.278 INFO:teuthology.orchestra.run.smithi040.stderr:/bin/podman: stderr: 2023-11-17T05:36:37.904+0000 7f79ffbc0700 1 -- start start 2023-11-17T05:36:40.278 INFO:teuthology.orchestra.run.smithi040.stderr:/bin/podman: stderr: 2023-11-17T05:36:37.905+0000 7f79ffbc0700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f79f80ff140 0x7f79f8060ba0 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:36:40.278 INFO:teuthology.orchestra.run.smithi040.stderr:/bin/podman: stderr: 2023-11-17T05:36:37.905+0000 7f79ffbc0700 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f79f80ffa30 0x7f79f8061090 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:36:40.278 INFO:teuthology.orchestra.run.smithi040.stderr:/bin/podman: 2023-11-17T05:36:37.905+0000 7f79ffbc0700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f79f80629f0 con 0x7f79f80ff140 2023-11-17T05:36:40.278 INFO:teuthology.orchestra.run.smithi040.stderr:/bin/podman: 2023-11-17T05:36:37.905+0000 7f79ffbc0700 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f79f8062b30 con 0x7f79f80ffa30 2023-11-17T05:36:40.278 INFO:teuthology.orchestra.run.smithi040.stderr:/bin/podman: 2023-11-17T05:36:37.905+0000 7f79fd95c700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f79f80ff140 0x7f79f8060ba0 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:36:40.278 INFO:teuthology.orchestra.run.smithi040.stderr:/bin/podman: stderr: 2023-11-17T05:36:37.905+0000 7f79fd95c700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f79f80ff140 0x7f79f8060ba0 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:52724/0 (socket says 172.21.15.40:52724) 2023-11-17T05:36:40.278 INFO:teuthology.orchestra.run.smithi040.stderr:/bin/podman: 2023-11-17T05:36:37.905+0000 7f79fd95c700 1 -- 172.21.15.40:0/799602753 learned_addr learned my addr 172.21.15.40:0/799602753 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-11-17T05:36:40.278 INFO:teuthology.orchestra.run.smithi040.stderr:/bin/podman: stderr: 2023-11-17T05:36:37.905+0000 7f79fd15b700 1 --2- 172.21.15.40:0/799602753 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f79f80ffa30 0x7f79f8061090 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:36:40.278 INFO:teuthology.orchestra.run.smithi040.stderr:/bin/podman: stderr: 2023-11-17T05:36:37.905+0000 7f79fd95c700 1 -- 172.21.15.40:0/799602753 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f79f80ffa30 msgr2=0x7f79f8061090 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:36:40.279 INFO:teuthology.orchestra.run.smithi040.stderr:/bin/podman: 2023-11-17T05:36:37.905+0000 7f79fd95c700 1 --2- 172.21.15.40:0/799602753 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f79f80ffa30 0x7f79f8061090 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:36:40.279 INFO:teuthology.orchestra.run.smithi040.stderr:/bin/podman: 2023-11-17T05:36:37.905+0000 7f79fd95c700 1 -- 172.21.15.40:0/799602753 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f79f8061580 con 0x7f79f80ff140 2023-11-17T05:36:40.279 INFO:teuthology.orchestra.run.smithi040.stderr:/bin/podman: stderr: 2023-11-17T05:36:37.905+0000 7f79fd95c700 1 --2- 172.21.15.40:0/799602753 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f79f80ff140 0x7f79f8060ba0 secure :-1 s=READY pgs=463 cs=0 l=1 rev1=1 rx=0x7f79f400e7c0 tx=0x7f79f400b040).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:36:40.279 INFO:teuthology.orchestra.run.smithi040.stderr:/bin/podman: stderr: 2023-11-17T05:36:37.905+0000 7f79ee7fc700 1 -- 172.21.15.40:0/799602753 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 357+0+0 (secure 0 0 0) 0x7f79f40092e0 con 0x7f79f80ff140 2023-11-17T05:36:40.279 INFO:teuthology.orchestra.run.smithi040.stderr:/bin/podman: 2023-11-17T05:36:37.905+0000 7f79ee7fc700 1 -- 172.21.15.40:0/799602753 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(25 keys) v1 ==== 1074+0+0 (secure 0 0 0) 0x7f79f400f040 con 0x7f79f80ff140 2023-11-17T05:36:40.279 INFO:teuthology.orchestra.run.smithi040.stderr:/bin/podman: stderr: 2023-11-17T05:36:37.905+0000 7f79ffbc0700 1 -- 172.21.15.40:0/799602753 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f79f8061830 con 0x7f79f80ff140 2023-11-17T05:36:40.279 INFO:teuthology.orchestra.run.smithi040.stderr:/bin/podman: 2023-11-17T05:36:37.905+0000 7f79ee7fc700 1 -- 172.21.15.40:0/799602753 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 357+0+0 (secure 0 0 0) 0x7f79f4014850 con 0x7f79f80ff140 2023-11-17T05:36:40.279 INFO:teuthology.orchestra.run.smithi040.stderr:/bin/podman: 2023-11-17T05:36:37.905+0000 7f79ffbc0700 1 -- 172.21.15.40:0/799602753 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f79f8118b50 con 0x7f79f80ff140 2023-11-17T05:36:40.297 INFO:teuthology.orchestra.run.smithi040.stderr:/bin/podman: stderr: 2023-11-17T05:36:37.907+0000 7f79ffbc0700 1 -- 172.21.15.40:0/799602753 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f79f8046570 con 0x7f79f80ff140 2023-11-17T05:36:40.297 INFO:teuthology.orchestra.run.smithi040.stderr:/bin/podman: stderr: 2023-11-17T05:36:37.907+0000 7f79ee7fc700 1 -- 172.21.15.40:0/799602753 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 18) v1 ==== 72193+0+0 (secure 0 0 0) 0x7f79f401b030 con 0x7f79f80ff140 2023-11-17T05:36:40.297 INFO:teuthology.orchestra.run.smithi040.stderr:/bin/podman: stderr: 2023-11-17T05:36:37.907+0000 7f79ee7fc700 1 --2- 172.21.15.40:0/799602753 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f79e405c370 0x7f79e405e830 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:36:40.297 INFO:teuthology.orchestra.run.smithi040.stderr:/bin/podman: stderr: 2023-11-17T05:36:37.907+0000 7f79ee7fc700 1 -- 172.21.15.40:0/799602753 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(6..6 src has 1..6) v4 ==== 1385+0+0 (secure 0 0 0) 0x7f79f4074e80 con 0x7f79f80ff140 2023-11-17T05:36:40.297 INFO:teuthology.orchestra.run.smithi040.stderr:/bin/podman: 2023-11-17T05:36:37.908+0000 7f79fd15b700 1 --2- 172.21.15.40:0/799602753 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f79e405c370 0x7f79e405e830 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:36:40.297 INFO:teuthology.orchestra.run.smithi040.stderr:/bin/podman: 2023-11-17T05:36:37.908+0000 7f79fd15b700 1 --2- 172.21.15.40:0/799602753 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f79e405c370 0x7f79e405e830 secure :-1 s=READY pgs=210 cs=0 l=1 rev1=1 rx=0x7f79e8004890 tx=0x7f79e800b040).ready entity=mgr.14164 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:36:40.297 INFO:teuthology.orchestra.run.smithi040.stderr:/bin/podman: stderr: 2023-11-17T05:36:37.911+0000 7f79ee7fc700 1 -- 172.21.15.40:0/799602753 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+152212 (secure 0 0 0) 0x7f79f4048ae0 con 0x7f79f80ff140 2023-11-17T05:36:40.298 INFO:teuthology.orchestra.run.smithi040.stderr:/bin/podman: stderr: 2023-11-17T05:36:38.142+0000 7f79ffbc0700 1 -- 172.21.15.40:0/799602753 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "mon getmap"} v 0) v1 -- 0x7f79f8108e40 con 0x7f79f80ff140 2023-11-17T05:36:40.298 INFO:teuthology.orchestra.run.smithi040.stderr:/bin/podman: stderr: 2023-11-17T05:36:38.142+0000 7f79ee7fc700 1 -- 172.21.15.40:0/799602753 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mon_command_ack([{"prefix": "mon getmap"}]=0 got monmap epoch 2 v2) v1 ==== 76+0+353 (secure 0 0 0) 0x7f79f4019030 con 0x7f79f80ff140 2023-11-17T05:36:40.298 INFO:teuthology.orchestra.run.smithi040.stderr:/bin/podman: stderr: 2023-11-17T05:36:38.145+0000 7f79ffbc0700 1 -- 172.21.15.40:0/799602753 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f79e405c370 msgr2=0x7f79e405e830 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:36:40.298 INFO:teuthology.orchestra.run.smithi040.stderr:/bin/podman: 2023-11-17T05:36:38.145+0000 7f79ffbc0700 1 --2- 172.21.15.40:0/799602753 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f79e405c370 0x7f79e405e830 secure :-1 s=READY pgs=210 cs=0 l=1 rev1=1 rx=0x7f79e8004890 tx=0x7f79e800b040).stop 2023-11-17T05:36:40.298 INFO:teuthology.orchestra.run.smithi040.stderr:/bin/podman: 2023-11-17T05:36:38.145+0000 7f79ffbc0700 1 -- 172.21.15.40:0/799602753 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f79f80ff140 msgr2=0x7f79f8060ba0 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:36:40.298 INFO:teuthology.orchestra.run.smithi040.stderr:/bin/podman: 2023-11-17T05:36:38.145+0000 7f79ffbc0700 1 --2- 172.21.15.40:0/799602753 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f79f80ff140 0x7f79f8060ba0 secure :-1 s=READY pgs=463 cs=0 l=1 rev1=1 rx=0x7f79f400e7c0 tx=0x7f79f400b040).stop 2023-11-17T05:36:40.298 INFO:teuthology.orchestra.run.smithi040.stderr:/bin/podman: stderr: 2023-11-17T05:36:38.145+0000 7f79ffbc0700 1 -- 172.21.15.40:0/799602753 shutdown_connections 2023-11-17T05:36:40.298 INFO:teuthology.orchestra.run.smithi040.stderr:/bin/podman: 2023-11-17T05:36:38.145+0000 7f79ffbc0700 1 --2- 172.21.15.40:0/799602753 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f79e405c370 0x7f79e405e830 unknown :-1 s=CLOSED pgs=210 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:36:40.298 INFO:teuthology.orchestra.run.smithi040.stderr:/bin/podman: 2023-11-17T05:36:38.145+0000 7f79ffbc0700 1 --2- 172.21.15.40:0/799602753 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f79f80ff140 0x7f79f8060ba0 unknown :-1 s=CLOSED pgs=463 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:36:40.298 INFO:teuthology.orchestra.run.smithi040.stderr:/bin/podman: 2023-11-17T05:36:38.145+0000 7f79ffbc0700 1 --2- 172.21.15.40:0/799602753 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f79f80ffa30 0x7f79f8061090 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:36:40.298 INFO:teuthology.orchestra.run.smithi040.stderr:/bin/podman: 2023-11-17T05:36:38.145+0000 7f79ffbc0700 1 -- 172.21.15.40:0/799602753 >> 172.21.15.40:0/799602753 conn(0x7f79f8065d40 msgr2=0x7f79f8103470 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:36:40.298 INFO:teuthology.orchestra.run.smithi040.stderr:/bin/podman: 2023-11-17T05:36:38.145+0000 7f79ffbc0700 1 -- 172.21.15.40:0/79 2023-11-17T05:36:40.298 INFO:teuthology.orchestra.run.smithi040.stderr:/bin/podman: stderr: 9602753 shutdown_connections 2023-11-17T05:36:40.299 INFO:teuthology.orchestra.run.smithi040.stderr:/bin/podman: 2023-11-17T05:36:38.145+0000 7f79ffbc0700 1 -- 172.21.15.40:0/799602753 wait complete. 2023-11-17T05:36:40.299 INFO:teuthology.orchestra.run.smithi040.stderr:/bin/podman: stderr: got monmap epoch 2 2023-11-17T05:36:40.299 INFO:teuthology.orchestra.run.smithi040.stderr:/bin/podman: Running command: /usr/bin/ceph-authtool /var/lib/ceph/osd/ceph-0/keyring --create-keyring --name osd.0 --add-key AQDl+1ZlqWAuGBAALBRW+VnVLCAu4Sv5GMlYRg== 2023-11-17T05:36:40.299 INFO:teuthology.orchestra.run.smithi040.stderr:/bin/podman: stdout: creating /var/lib/ceph/osd/ceph-0/keyring 2023-11-17T05:36:40.299 INFO:teuthology.orchestra.run.smithi040.stderr:/bin/podman: added entity osd.0 auth(key=AQDl+1ZlqWAuGBAALBRW+VnVLCAu4Sv5GMlYRg==) 2023-11-17T05:36:40.299 INFO:teuthology.orchestra.run.smithi040.stderr:/bin/podman: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0/keyring 2023-11-17T05:36:40.299 INFO:teuthology.orchestra.run.smithi040.stderr:/bin/podman: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0/ 2023-11-17T05:36:40.299 INFO:teuthology.orchestra.run.smithi040.stderr:/bin/podman: Running command: /usr/bin/ceph-osd --cluster ceph --osd-objectstore bluestore --mkfs -i 0 --monmap /var/lib/ceph/osd/ceph-0/activate.monmap --keyfile - --osdspec-affinity None --osd-data /var/lib/ceph/osd/ceph-0/ --osd-uuid 6153a2e4-5c5c-4ea4-85b8-fbcee0397d79 --setuser ceph --setgroup ceph 2023-11-17T05:36:40.299 INFO:teuthology.orchestra.run.smithi040.stderr:/bin/podman: stderr: 2023-11-17T05:36:38.203+0000 7f854fc87080 -1 bluestore(/var/lib/ceph/osd/ceph-0/) _read_fsid unparsable uuid 2023-11-17T05:36:40.299 INFO:teuthology.orchestra.run.smithi040.stderr:/bin/podman: stderr: 2023-11-17T05:36:38.563+0000 7f854fc87080 -1 bluefs _replay 0x0: stop: uuid 00000000-0000-0000-0000-000000000000 != super.uuid 0ffb7a6f-1ef2-4d96-af47-8d61bbdc0809, block dump: 2023-11-17T05:36:40.299 INFO:teuthology.orchestra.run.smithi040.stderr:/bin/podman: stderr: 00000000 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................| 2023-11-17T05:36:40.299 INFO:teuthology.orchestra.run.smithi040.stderr:/bin/podman: stderr: * 2023-11-17T05:36:40.299 INFO:teuthology.orchestra.run.smithi040.stderr:/bin/podman: stderr: 00000ff0 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 00 |................| 2023-11-17T05:36:40.300 INFO:teuthology.orchestra.run.smithi040.stderr:/bin/podman: stderr: 00001000 2023-11-17T05:36:40.300 INFO:teuthology.orchestra.run.smithi040.stderr:/bin/podman: stderr: 2023-11-17T05:36:38.974+0000 7f854fc87080 -1 bluestore(/var/lib/ceph/osd/ceph-0/) _open_db erroring opening db: 2023-11-17T05:36:40.300 INFO:teuthology.orchestra.run.smithi040.stderr:/bin/podman: stderr: 2023-11-17T05:36:39.486+0000 7f854fc87080 -1 OSD::mkfs: ObjectStore::mkfs failed with error (5) Input/output error 2023-11-17T05:36:40.300 INFO:teuthology.orchestra.run.smithi040.stderr:/bin/podman: stderr: 2023-11-17T05:36:39.486+0000 7f854fc87080 -1  ** ERROR: error creating empty object store in /var/lib/ceph/osd/ceph-0/: (5) Input/output error 2023-11-17T05:36:40.300 INFO:teuthology.orchestra.run.smithi040.stderr:/bin/podman: --> Was unable to complete a new OSD, will rollback changes 2023-11-17T05:36:40.300 INFO:teuthology.orchestra.run.smithi040.stderr:/bin/podman: Running command: /usr/bin/ceph --cluster ceph --name client.bootstrap-osd --keyring /var/lib/ceph/bootstrap-osd/ceph.keyring osd purge-new osd.0 --yes-i-really-mean-it 2023-11-17T05:36:40.300 INFO:teuthology.orchestra.run.smithi040.stderr:/bin/podman: stderr: 2023-11-17T05:36:39.601+0000 7f9603047700 1 -- 172.21.15.40:0/3178258714 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f95fc0ffa10 msgr2=0x7f95fc105c50 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:36:40.300 INFO:teuthology.orchestra.run.smithi040.stderr:/bin/podman: 2023-11-17T05:36:39.601+0000 7f9603047700 1 --2- 172.21.15.40:0/3178258714 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f95fc0ffa10 0x7f95fc105c50 secure :-1 s=READY pgs=465 cs=0 l=1 rev1=1 rx=0x7f95e80048f0 tx=0x7f95e8009b30).stop 2023-11-17T05:36:40.300 INFO:teuthology.orchestra.run.smithi040.stderr:/bin/podman: 2023-11-17T05:36:39.601+0000 7f9603047700 1 -- 172.21.15.40:0/3178258714 shutdown_connections 2023-11-17T05:36:40.300 INFO:teuthology.orchestra.run.smithi040.stderr:/bin/podman: 2023-11-17T05:36:39.601+0000 7f9603047700 1 --2- 172.21.15.40:0/3178258714 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f95fc0ffa10 0x7f95fc105c50 unknown :-1 s=CLOSED pgs=465 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:36:40.300 INFO:teuthology.orchestra.run.smithi040.stderr:/bin/podman: 2023-11-17T05:36:39.601+0000 7f9603047700 1 --2- 172.21.15.40:0/3178258714 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f95fc0ff120 0x7f95fc0ff520 unknown :-1 s=CLOSED pgs=0 cs=0 l=0 rev1=1 rx=0 tx=0).stop 2023-11-17T05:36:40.300 INFO:teuthology.orchestra.run.smithi040.stderr:/bin/podman: 2023-11-17T05: 2023-11-17T05:36:40.300 INFO:teuthology.orchestra.run.smithi040.stderr:/bin/podman: stderr: 36:39.601+0000 7f9603047700 1 -- 172.21.15.40:0/3178258714 >> 172.21.15.40:0/3178258714 conn(0x7f95fc065d90 msgr2=0x7f95fc0681b0 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:36:40.300 INFO:teuthology.orchestra.run.smithi040.stderr:/bin/podman: 2023-11-17T05:36:39.601+0000 7f9603047700 1 -- 172.21.15.40:0/3178258714 shutdown_connections 2023-11-17T05:36:40.301 INFO:teuthology.orchestra.run.smithi040.stderr:/bin/podman: 2023-11-17T05:36:39.601+0000 7f9603047700 1 -- 172.21.15.40:0/3178258714 wait complete. 2023-11-17T05:36:40.301 INFO:teuthology.orchestra.run.smithi040.stderr:/bin/podman: stderr: 2023-11-17T05:36:39.601+0000 7f9603047700 1 Processor -- start 2023-11-17T05:36:40.301 INFO:teuthology.orchestra.run.smithi040.stderr:/bin/podman: stderr: 2023-11-17T05:36:39.601+0000 7f9603047700 1 -- start start 2023-11-17T05:36:40.301 INFO:teuthology.orchestra.run.smithi040.stderr:/bin/podman: stderr: 2023-11-17T05:36:39.602+0000 7f9603047700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f95fc0ff120 0x7f95fc112320 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:36:40.301 INFO:teuthology.orchestra.run.smithi040.stderr:/bin/podman: 2023-11-17T05:36:39.602+0000 7f9603047700 1 --2- >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f95fc0ffa10 0x7f95fc112810 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:36:40.301 INFO:teuthology.orchestra.run.smithi040.stderr:/bin/podman: stderr: 2023-11-17T05:36:39.602+0000 7f9603047700 1 -- --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f95fc112e10 con 0x7f95fc0ff120 2023-11-17T05:36:40.301 INFO:teuthology.orchestra.run.smithi040.stderr:/bin/podman: 2023-11-17T05:36:39.602+0000 7f9603047700 1 -- --> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] -- mon_getmap magic: 0 v1 -- 0x7f95fc10f580 con 0x7f95fc0ffa10 2023-11-17T05:36:40.301 INFO:teuthology.orchestra.run.smithi040.stderr:/bin/podman: stderr: 2023-11-17T05:36:39.602+0000 7f9600de3700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f95fc0ff120 0x7f95fc112320 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:36:40.301 INFO:teuthology.orchestra.run.smithi040.stderr:/bin/podman: stderr: 2023-11-17T05:36:39.602+0000 7f9600de3700 1 --2- >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f95fc0ff120 0x7f95fc112320 unknown :-1 s=HELLO_CONNECTING pgs=0 cs=0 l=1 rev1=1 rx=0 tx=0).handle_hello peer v2:172.21.15.40:3300/0 says I am v2:172.21.15.40:52746/0 (socket says 172.21.15.40:52746) 2023-11-17T05:36:40.301 INFO:teuthology.orchestra.run.smithi040.stderr:/bin/podman: 2023-11-17T05:36:39.602+0000 7f9600de3700 1 -- 172.21.15.40:0/1216118669 learned_addr learned my addr 172.21.15.40:0/1216118669 (peer_addr_for_me v2:172.21.15.40:0/0) 2023-11-17T05:36:40.301 INFO:teuthology.orchestra.run.smithi040.stderr:/bin/podman: stderr: 2023-11-17T05:36:39.602+0000 7f95fbfff700 1 --2- 172.21.15.40:0/1216118669 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f95fc0ffa10 0x7f95fc112810 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:36:40.301 INFO:teuthology.orchestra.run.smithi040.stderr:/bin/podman: stderr: 2023-11-17T05:36:39.602+0000 7f9600de3700 1 -- 172.21.15.40:0/1216118669 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f95fc0ffa10 msgr2=0x7f95fc112810 unknown :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:36:40.301 INFO:teuthology.orchestra.run.smithi040.stderr:/bin/podman: 2023-11-17T05:36:39.602+0000 7f9600de3700 1 --2- 172.21.15.40:0/1216118669 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f95fc0ffa10 0x7f95fc112810 unknown :-1 s=AUTH_CONNECTING pgs=0 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:36:40.301 INFO:teuthology.orchestra.run.smithi040.stderr:/bin/podman: 2023-11-17T05:36:39.602+0000 7f9600de3700 1 -- 172.21.15.40:0/1216118669 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({config=0+,monmap=0+}) v3 -- 0x7f95fc10f740 con 0x7f95fc0ff120 2023-11-17T05:36:40.302 INFO:teuthology.orchestra.run.smithi040.stderr:/bin/podman: 2023-11-17T05:36:39.602+0000 7f95fbfff700 1 --2- 172.21.15.40:0/1216118669 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f95fc0ffa10 0x7f95fc112810 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 rx=0 tx=0).send_auth_request state changed! 2023-11-17T05:36:40.302 INFO:teuthology.orchestra.run.smithi040.stderr:/bin/podman: stderr: 2023-11-17T05:36:39.602+0000 7f9600de3700 1 --2- 172.21.15.40:0/1216118669 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f95fc0ff120 0x7f95fc112320 secure :-1 s=READY pgs=466 cs=0 l=1 rev1=1 rx=0x7f95f000c580 tx=0x7f95f0007e40).ready entity=mon.0 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:36:40.302 INFO:teuthology.orchestra.run.smithi040.stderr:/bin/podman: stderr: 2023-11-17T05:36:39.603+0000 7f95f97fa700 1 -- 172.21.15.40:0/1216118669 <== mon.0 v2:172.21.15.40:3300/0 1 ==== mon_map magic: 0 v1 ==== 357+0+0 (secure 0 0 0) 0x7f95f000e680 con 0x7f95fc0ff120 2023-11-17T05:36:40.302 INFO:teuthology.orchestra.run.smithi040.stderr:/bin/podman: stderr: 2023-11-17T05:36:39.603+0000 7f9603047700 1 -- 172.21.15.40:0/1216118669 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({mgrmap=0+}) v3 -- 0x7f95fc10f9f0 con 0x7f95fc0ff120 2023-11-17T05:36:40.302 INFO:teuthology.orchestra.run.smithi040.stderr:/bin/podman: 2023-11-17T05:36:39.603+0000 7f95f97fa700 1 -- 172.21.15.40:0/1216118669 <== mon.0 v2:172.21.15.40:3300/0 2 ==== config(25 keys) v1 ==== 1074+0+0 (secure 0 0 0) 0x7f95f000ec90 con 0x7f95fc0ff120 2023-11-17T05:36:40.302 INFO:teuthology.orchestra.run.smithi040.stderr:/bin/podman: 2023-11-17T05:36:39.603+0000 7f95f97fa700 1 -- 172.21.15.40:0/1216118669 <== mon.0 v2:172.21.15.40:3300/0 3 ==== mon_map magic: 0 v1 ==== 357+0+0 (secure 0 0 0) 0x7f95f0012ca0 con 0x7f95fc0ff120 2023-11-17T05:36:40.302 INFO:teuthology.orchestra.run.smithi040.stderr:/bin/podman: stderr: 2023-11-17T05:36:39.603+0000 7f9603047700 1 -- 172.21.15.40:0/1216118669 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_subscribe({osdmap=0}) v3 -- 0x7f95fc10ff30 con 0x7f95fc0ff120 2023-11-17T05:36:40.302 INFO:teuthology.orchestra.run.smithi040.stderr:/bin/podman: stderr: 2023-11-17T05:36:39.604+0000 7f9603047700 1 -- 172.21.15.40:0/1216118669 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "get_command_descriptions"} v 0) v1 -- 0x7f95fc001550 con 0x7f95fc0ff120 2023-11-17T05:36:40.302 INFO:teuthology.orchestra.run.smithi040.stderr:/bin/podman: stderr: 2023-11-17T05:36:39.604+0000 7f95f97fa700 1 -- 172.21.15.40:0/1216118669 <== mon.0 v2:172.21.15.40:3300/0 4 ==== mgrmap(e 18) v1 ==== 72193+0+0 (secure 0 0 0) 0x7f95f0019030 con 0x7f95fc0ff120 2023-11-17T05:36:40.315 INFO:teuthology.orchestra.run.smithi040.stderr:/bin/podman: stderr: 2023-11-17T05:36:39.605+0000 7f95f97fa700 1 --2- 172.21.15.40:0/1216118669 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f95ec05c360 0x7f95ec05e820 unknown :-1 s=NONE pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0).connect 2023-11-17T05:36:40.315 INFO:teuthology.orchestra.run.smithi040.stderr:/bin/podman: stderr: 2023-11-17T05:36:39.605+0000 7f95f97fa700 1 -- 172.21.15.40:0/1216118669 <== mon.0 v2:172.21.15.40:3300/0 5 ==== osd_map(6..6 src has 1..6) v4 ==== 1385+0+0 (secure 0 0 0) 0x7f95f0074c90 con 0x7f95fc0ff120 2023-11-17T05:36:40.315 INFO:teuthology.orchestra.run.smithi040.stderr:/bin/podman: 2023-11-17T05:36:39.605+0000 7f95fbfff700 1 --2- 172.21.15.40:0/1216118669 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f95ec05c360 0x7f95ec05e820 unknown :-1 s=BANNER_CONNECTING pgs=0 cs=0 l=1 rev1=0 rx=0 tx=0)._handle_peer_banner_payload supported=1 required=0 2023-11-17T05:36:40.315 INFO:teuthology.orchestra.run.smithi040.stderr:/bin/podman: stderr: 2023-11-17T05:36:39.605+0000 7f95fbfff700 1 --2- 172.21.15.40:0/1216118669 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f95ec05c360 0x7f95ec05e820 secure :-1 s=READY pgs=211 cs=0 l=1 rev1=1 rx=0x7f95e800b370 tx=0x7f95e8009b30).ready entity=mgr.14164 client_cookie=0 server_cookie=0 in_seq=0 out_seq=0 2023-11-17T05:36:40.315 INFO:teuthology.orchestra.run.smithi040.stderr:/bin/podman: stderr: 2023-11-17T05:36:39.609+0000 7f95f97fa700 1 -- 172.21.15.40:0/1216118669 <== mon.0 v2:172.21.15.40:3300/0 6 ==== mon_command_ack([{"prefix": "get_command_descriptions"}]=0 v0) v1 ==== 72+0+152212 (secure 0 0 0) 0x7f95f00462b0 con 0x7f95fc0ff120 2023-11-17T05:36:40.315 INFO:teuthology.orchestra.run.smithi040.stderr:/bin/podman: stderr: 2023-11-17T05:36:39.787+0000 7f9603047700 1 -- 172.21.15.40:0/1216118669 --> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] -- mon_command({"prefix": "osd purge-new", "id": 0, "yes_i_really_mean_it": true} v 0) v1 -- 0x7f95fc108e00 con 0x7f95fc0ff120 2023-11-17T05:36:40.315 INFO:teuthology.orchestra.run.smithi040.stderr:/bin/podman: stderr: 2023-11-17T05:36:39.792+0000 7f95f97fa700 1 -- 172.21.15.40:0/1216118669 <== mon.0 v2:172.21.15.40:3300/0 7 ==== mon_command_ack([{"prefix": "osd purge-new", "id": 0, "yes_i_really_mean_it": true}]=0 purged osd.0 v7) v1 ==== 112+0+0 (secure 0 0 0) 0x7f95f0017030 con 0x7f95fc0ff120 2023-11-17T05:36:40.316 INFO:teuthology.orchestra.run.smithi040.stderr:/bin/podman: stderr: 2023-11-17T05:36:39.793+0000 7f9603047700 1 -- 172.21.15.40:0/1216118669 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f95ec05c360 msgr2=0x7f95ec05e820 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:36:40.316 INFO:teuthology.orchestra.run.smithi040.stderr:/bin/podman: 2023-11-17T05:36:39.793+0000 7f9603047700 1 --2- 172.21.15.40:0/1216118669 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f95ec05c360 0x7f95ec05e820 secure :-1 s=READY pgs=211 cs=0 l=1 rev1=1 rx=0x7f95e800b370 tx=0x7f95e8009b30).stop 2023-11-17T05:36:40.316 INFO:teuthology.orchestra.run.smithi040.stderr:/bin/podman: 2023-11-17T05:36:39.793+0000 7f9603047700 1 -- 172.21.15.40:0/1216118669 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f95fc0ff120 msgr2=0x7f95fc112320 secure :-1 s=STATE_CONNECTION_ESTABLISHED l=1).mark_down 2023-11-17T05:36:40.316 INFO:teuthology.orchestra.run.smithi040.stderr:/bin/podman: 2023-11-17T05:36:39.793+0000 7f9603047700 1 --2- 172.21.15.40:0/1216118669 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f95fc0ff120 0x7f95fc112320 secure :-1 s=READY pgs=466 cs=0 l=1 rev1=1 rx=0x7f95f000c580 tx=0x7f95f0007e40).stop 2023-11-17T05:36:40.316 INFO:teuthology.orchestra.run.smithi040.stderr:/bin/podman: stderr: 2023-11-17T05:36:39.794+0000 7f9603047700 1 -- 172.21.15.40:0/1216118669 shutdown_connections 2023-11-17T05:36:40.316 INFO:teuthology.orchestra.run.smithi040.stderr:/bin/podman: 2023-11-17T05:36:39.794+0000 7f9603047700 1 --2- 172.21.15.40:0/1216118669 >> [v2:172.21.15.40:6800/1119389001,v1:172.21.15.40:6801/1119389001] conn(0x7f95ec05c360 0x7f95ec05e820 unknown :-1 s=CLOSED pgs=211 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:36:40.316 INFO:teuthology.orchestra.run.smithi040.stderr:/bin/podman: 2023-11-17T05:36:39.794+0000 7f9603047700 1 --2- 172.21.15.40:0/1216118669 >> [v2:172.21.15.40:3300/0,v1:172.21.15.40:6789/0] conn(0x7f95fc0ff120 0x7f95fc112320 unknown :-1 s=CLOSED pgs=466 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:36:40.316 INFO:teuthology.orchestra.run.smithi040.stderr:/bin/podman: 2023-11-17T05:36:39.794+0000 7f9603047700 1 --2- 172.21.15.40:0/1216118669 >> [v2:172.21.15.66:3300/0,v1:172.21.15.66:6789/0] conn(0x7f95fc0ffa10 0x7f95fc112810 unknown :-1 s=CLOSED pgs=0 cs=0 l=1 rev1=1 rx=0 tx=0).stop 2023-11-17T05:36:40.316 INFO:teuthology.orchestra.run.smithi040.stderr:/bin/podman: 2023-11-17T05:36:39.794+0000 7f9603047700 1 -- 172.21.15.40:0/1216118669 >> 172.21.15.40:0/1216118669 conn(0x7f95fc065d90 msgr2=0x7f95fc103450 unknown :-1 s=STATE_NONE l=0).mark_down 2023-11-17T05:36:40.316 INFO:teuthology.orchestra.run.smithi040.stderr:/bin/podman: 2023-11-17T05:36:39.794+0000 7f9603047700 1 -- 172.21.15.4 2023-11-17T05:36:40.316 INFO:teuthology.orchestra.run.smithi040.stderr:/bin/podman: stderr: 0:0/1216118669 shutdown_connections 2023-11-17T05:36:40.316 INFO:teuthology.orchestra.run.smithi040.stderr:/bin/podman: 2023-11-17T05:36:39.794+0000 7f9603047700 1 -- 172.21.15.40:0/1216118669 wait complete. 2023-11-17T05:36:40.316 INFO:teuthology.orchestra.run.smithi040.stderr:/bin/podman: stderr: purged osd.0 2023-11-17T05:36:40.316 INFO:teuthology.orchestra.run.smithi040.stderr:/bin/podman: --> RuntimeError: Command failed with exit code 250: /usr/bin/ceph-osd --cluster ceph --osd-objectstore bluestore --mkfs -i 0 --monmap /var/lib/ceph/osd/ceph-0/activate.monmap --keyfile - --osdspec-affinity None --osd-data /var/lib/ceph/osd/ceph-0/ --osd-uuid 6153a2e4-5c5c-4ea4-85b8-fbcee0397d79 --setuser ceph --setgroup ceph 2023-11-17T05:36:40.317 INFO:teuthology.orchestra.run.smithi040.stderr:Traceback (most recent call last): 2023-11-17T05:36:40.317 INFO:teuthology.orchestra.run.smithi040.stderr: File "/var/lib/ceph/ed6470e2-8508-11ee-8dbe-212e2dc638e7/cephadm.30cb78bdbbafb384af862e1c2292b944f15942b586128e91262b43e91e11ae90", line 8029, in 2023-11-17T05:36:40.317 INFO:teuthology.orchestra.run.smithi040.stderr: main() 2023-11-17T05:36:40.317 INFO:teuthology.orchestra.run.smithi040.stderr: File "/var/lib/ceph/ed6470e2-8508-11ee-8dbe-212e2dc638e7/cephadm.30cb78bdbbafb384af862e1c2292b944f15942b586128e91262b43e91e11ae90", line 8017, in main 2023-11-17T05:36:40.317 INFO:teuthology.orchestra.run.smithi040.stderr: r = ctx.func(ctx) 2023-11-17T05:36:40.317 INFO:teuthology.orchestra.run.smithi040.stderr: File "/var/lib/ceph/ed6470e2-8508-11ee-8dbe-212e2dc638e7/cephadm.30cb78bdbbafb384af862e1c2292b944f15942b586128e91262b43e91e11ae90", line 1654, in _infer_fsid 2023-11-17T05:36:40.317 INFO:teuthology.orchestra.run.smithi040.stderr: return func(ctx) 2023-11-17T05:36:40.317 INFO:teuthology.orchestra.run.smithi040.stderr: File "/var/lib/ceph/ed6470e2-8508-11ee-8dbe-212e2dc638e7/cephadm.30cb78bdbbafb384af862e1c2292b944f15942b586128e91262b43e91e11ae90", line 1738, in _infer_image 2023-11-17T05:36:40.317 INFO:teuthology.orchestra.run.smithi040.stderr: return func(ctx) 2023-11-17T05:36:40.317 INFO:teuthology.orchestra.run.smithi040.stderr: File "/var/lib/ceph/ed6470e2-8508-11ee-8dbe-212e2dc638e7/cephadm.30cb78bdbbafb384af862e1c2292b944f15942b586128e91262b43e91e11ae90", line 4514, in command_ceph_volume 2023-11-17T05:36:40.317 INFO:teuthology.orchestra.run.smithi040.stderr: out, err, code = call_throws(ctx, c.run_cmd(), verbosity=verbosity) 2023-11-17T05:36:40.319 INFO:teuthology.orchestra.run.smithi040.stderr: File "/var/lib/ceph/ed6470e2-8508-11ee-8dbe-212e2dc638e7/cephadm.30cb78bdbbafb384af862e1c2292b944f15942b586128e91262b43e91e11ae90", line 1464, in call_throws 2023-11-17T05:36:40.319 INFO:teuthology.orchestra.run.smithi040.stderr: raise RuntimeError('Failed command: %s' % ' '.join(command)) 2023-11-17T05:36:40.319 INFO:teuthology.orchestra.run.smithi040.stderr:RuntimeError: Failed command: /bin/podman run --rm --ipc=host --no-hosts --net=host --entrypoint /usr/sbin/ceph-volume --privileged --group-add=disk --init -e CONTAINER_IMAGE=docker.io/ceph/ceph@sha256:54e95ae1e11404157d7b329d0bef866ebbb214b195a009e87aae4eba9d282949 -e NODE_NAME=smithi040 -e CEPH_USE_RANDOM_NONCE=1 -e CEPH_VOLUME_OSDSPEC_AFFINITY=None -v /var/run/ceph/ed6470e2-8508-11ee-8dbe-212e2dc638e7:/var/run/ceph:z -v /var/log/ceph/ed6470e2-8508-11ee-8dbe-212e2dc638e7:/var/log/ceph:z -v /var/lib/ceph/ed6470e2-8508-11ee-8dbe-212e2dc638e7/crash:/var/lib/ceph/crash:z -v /dev:/dev -v /run/udev:/run/udev -v /sys:/sys -v /var/lib/ceph/ed6470e2-8508-11ee-8dbe-212e2dc638e7/selinux:/sys/fs/selinux:ro -v /run/lvm:/run/lvm -v /run/lock/lvm:/run/lock/lvm -v /tmp/ceph-tmpz30t5yd6:/etc/ceph/ceph.conf:z -v /tmp/ceph-tmpgoj2newa:/var/lib/ceph/bootstrap-osd/ceph.keyring:z docker.io/ceph/ceph@sha256:54e95ae1e11404157d7b329d0bef866ebbb214b195a009e87aae4eba9d282949 lvm batch --no-auto vg_nvme/lv_4 --yes --no-systemd 2023-11-17T05:36:40.320 INFO:teuthology.orchestra.run.smithi040.stderr: 2023-11-17T05:36:40.714 DEBUG:teuthology.orchestra.run:got remote process result: 22 2023-11-17T05:36:40.715 ERROR:teuthology.contextutil:Saw exception from nested tasks Traceback (most recent call last): File "/home/teuthworker/src/git.ceph.com_teuthology_6899cd26fceddb2fec83dc1a1349394b28c8998e/teuthology/contextutil.py", line 30, in nested vars.append(enter()) File "/usr/lib/python3.8/contextlib.py", line 113, in __enter__ return next(self.gen) File "/home/teuthworker/src/github.com_ceph_ceph-c_ed3c8e926b9de93026c8a7c1d86be5d6821b52ac/qa/tasks/cephadm.py", line 752, in ceph_osds _shell(ctx, cluster_name, remote, [ File "/home/teuthworker/src/github.com_ceph_ceph-c_ed3c8e926b9de93026c8a7c1d86be5d6821b52ac/qa/tasks/cephadm.py", line 34, in _shell return remote.run( File "/home/teuthworker/src/git.ceph.com_teuthology_6899cd26fceddb2fec83dc1a1349394b28c8998e/teuthology/orchestra/remote.py", line 522, in run r = self._runner(client=self.ssh, name=self.shortname, **kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_6899cd26fceddb2fec83dc1a1349394b28c8998e/teuthology/orchestra/run.py", line 455, in run r.wait() File "/home/teuthworker/src/git.ceph.com_teuthology_6899cd26fceddb2fec83dc1a1349394b28c8998e/teuthology/orchestra/run.py", line 161, in wait self._raise_for_status() File "/home/teuthworker/src/git.ceph.com_teuthology_6899cd26fceddb2fec83dc1a1349394b28c8998e/teuthology/orchestra/run.py", line 181, in _raise_for_status raise CommandFailedError( teuthology.exceptions.CommandFailedError: Command failed on smithi040 with status 22: 'sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.4 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ed6470e2-8508-11ee-8dbe-212e2dc638e7 -- ceph orch daemon add osd smithi040:vg_nvme/lv_4' 2023-11-17T05:36:40.717 DEBUG:teuthology.orchestra.run.smithi040:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2023-11-17T05:36:40.738 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:36:40 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:36:39.568790+0000 mgr.smithi040.ogcprl (mgr.14164) 482 : cluster [DBG] pgmap v460: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:36:40.739 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:36:40 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:36:39.789627+0000 mon.smithi040 (mon.0) 392 : audit [INF] from='client.? 172.21.15.40:0/1216118669' entity='client.bootstrap-osd' cmd=[{"prefix": "osd purge-new", "id": 0, "yes_i_really_mean_it": true}]: dispatch 2023-11-17T05:36:40.739 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:36:40 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: audit 2023-11-17T05:36:39.793635+0000 mon.smithi040 (mon.0) 393 : audit [INF] from='client.? 172.21.15.40:0/1216118669' entity='client.bootstrap-osd' cmd='[{"prefix": "osd purge-new", "id": 0, "yes_i_really_mean_it": true}]': finished 2023-11-17T05:36:40.739 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:36:40 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: cluster 2023-11-17T05:36:39.793697+0000 mon.smithi040 (mon.0) 394 : cluster [DBG] osdmap e7: 0 total, 0 up, 0 in 2023-11-17T05:36:40.755 DEBUG:teuthology.orchestra.run.smithi066:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2023-11-17T05:36:40.790 INFO:tasks.cephadm:Cleaning up testdir ceph.* files... 2023-11-17T05:36:40.790 DEBUG:teuthology.orchestra.run.smithi040:> rm -f /home/ubuntu/cephtest/seed.ceph.conf /home/ubuntu/cephtest/ceph.pub 2023-11-17T05:36:40.799 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:40 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: cluster 2023-11-17T05:36:39.568790+0000 mgr.smithi040.ogcprl (mgr.14164) 482 : cluster [DBG] pgmap v460: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-11-17T05:36:40.800 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:40 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: audit 2023-11-17T05:36:39.789627+0000 mon.smithi040 (mon.0) 392 : audit [INF] from='client.? 172.21.15.40:0/1216118669' entity='client.bootstrap-osd' cmd=[{"prefix": "osd purge-new", "id": 0, "yes_i_really_mean_it": true}]: dispatch 2023-11-17T05:36:40.800 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:40 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: audit 2023-11-17T05:36:39.793635+0000 mon.smithi040 (mon.0) 393 : audit [INF] from='client.? 172.21.15.40:0/1216118669' entity='client.bootstrap-osd' cmd='[{"prefix": "osd purge-new", "id": 0, "yes_i_really_mean_it": true}]': finished 2023-11-17T05:36:40.800 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:40 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: cluster 2023-11-17T05:36:39.793697+0000 mon.smithi040 (mon.0) 394 : cluster [DBG] osdmap e7: 0 total, 0 up, 0 in 2023-11-17T05:36:40.820 DEBUG:teuthology.orchestra.run.smithi066:> rm -f /home/ubuntu/cephtest/seed.ceph.conf /home/ubuntu/cephtest/ceph.pub 2023-11-17T05:36:40.841 INFO:tasks.cephadm:Stopping all daemons... 2023-11-17T05:36:40.841 INFO:tasks.cephadm.mon.smithi040:Stopping mon.smithi040... 2023-11-17T05:36:40.842 DEBUG:teuthology.orchestra.run.smithi040:> sudo systemctl stop ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7@mon.smithi040 2023-11-17T05:36:41.201 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:36:40 smithi040 systemd[1]: Stopping Ceph mon.smithi040 for ed6470e2-8508-11ee-8dbe-212e2dc638e7... 2023-11-17T05:36:41.201 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:36:40 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: debug 2023-11-17T05:36:40.995+0000 7fab8fd8d700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.smithi040 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-stderr=true (PID: 1) UID: 0 2023-11-17T05:36:41.201 INFO:journalctl@ceph.mon.smithi040.smithi040.stdout:Nov 17 05:36:40 smithi040 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi040[111758]: debug 2023-11-17T05:36:40.995+0000 7fab8fd8d700 -1 mon.smithi040@0(leader) e2 *** Got Signal Terminated *** 2023-11-17T05:36:41.370 DEBUG:teuthology.orchestra.run.smithi040:> sudo pkill -f 'journalctl -f -n 0 -u ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7@mon.smithi040.service' 2023-11-17T05:36:41.440 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-11-17T05:36:41.441 INFO:tasks.cephadm.mon.smithi040:Stopped mon.smithi040 2023-11-17T05:36:41.441 INFO:tasks.cephadm.mon.smithi066:Stopping mon.smithi066... 2023-11-17T05:36:41.441 DEBUG:teuthology.orchestra.run.smithi066:> sudo systemctl stop ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7@mon.smithi066 2023-11-17T05:36:41.800 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:41 smithi066 systemd[1]: Stopping Ceph mon.smithi066 for ed6470e2-8508-11ee-8dbe-212e2dc638e7... 2023-11-17T05:36:41.800 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:41 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: debug 2023-11-17T05:36:41.581+0000 7f70ef89d700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.smithi066 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-stderr=true (PID: 1) UID: 0 2023-11-17T05:36:41.800 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:41 smithi066 ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066[145378]: debug 2023-11-17T05:36:41.581+0000 7f70ef89d700 -1 mon.smithi066@1(peon) e2 *** Got Signal Terminated *** 2023-11-17T05:36:42.235 DEBUG:teuthology.orchestra.run.smithi066:> sudo pkill -f 'journalctl -f -n 0 -u ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7@mon.smithi066.service' 2023-11-17T05:36:42.263 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:42 smithi066 podman[146236]: ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7-mon.smithi066 2023-11-17T05:36:42.263 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:42 smithi066 systemd[1]: ceph-ed6470e2-8508-11ee-8dbe-212e2dc638e7@mon.smithi066.service: Succeeded. 2023-11-17T05:36:42.263 INFO:journalctl@ceph.mon.smithi066.smithi066.stdout:Nov 17 05:36:42 smithi066 systemd[1]: Stopped Ceph mon.smithi066 for ed6470e2-8508-11ee-8dbe-212e2dc638e7. 2023-11-17T05:36:42.283 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-11-17T05:36:42.283 INFO:tasks.cephadm.mon.smithi066:Stopped mon.smithi066 2023-11-17T05:36:42.283 DEBUG:teuthology.orchestra.run.smithi040:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid ed6470e2-8508-11ee-8dbe-212e2dc638e7 --force --keep-logs 2023-11-17T05:36:44.513 DEBUG:teuthology.orchestra.run.smithi066:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid ed6470e2-8508-11ee-8dbe-212e2dc638e7 --force --keep-logs 2023-11-17T05:38:16.819 DEBUG:teuthology.orchestra.run.smithi040:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2023-11-17T05:38:16.862 DEBUG:teuthology.orchestra.run.smithi066:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2023-11-17T05:38:16.893 INFO:tasks.cephadm:Archiving crash dumps... 2023-11-17T05:38:16.896 DEBUG:teuthology.misc:Transferring archived files from smithi040:/var/lib/ceph/ed6470e2-8508-11ee-8dbe-212e2dc638e7/crash to /home/teuthworker/archive/yuriw-2023-11-16_22:29:26-rados-wip-yuri3-testing-2023-11-14-1227-pacific-distro-default-smithi/7461274/remote/ubuntu@smithi040.front.sepia.ceph.com/crash 2023-11-17T05:38:16.898 DEBUG:teuthology.orchestra.run.smithi040:> sudo tar c -f - -C /var/lib/ceph/ed6470e2-8508-11ee-8dbe-212e2dc638e7/crash -- . 2023-11-17T05:38:16.944 INFO:teuthology.orchestra.run.smithi040.stderr:tar: /var/lib/ceph/ed6470e2-8508-11ee-8dbe-212e2dc638e7/crash: Cannot open: No such file or directory 2023-11-17T05:38:16.945 INFO:teuthology.orchestra.run.smithi040.stderr:tar: Error is not recoverable: exiting now 2023-11-17T05:38:16.948 DEBUG:teuthology.misc:Transferring archived files from smithi066:/var/lib/ceph/ed6470e2-8508-11ee-8dbe-212e2dc638e7/crash to /home/teuthworker/archive/yuriw-2023-11-16_22:29:26-rados-wip-yuri3-testing-2023-11-14-1227-pacific-distro-default-smithi/7461274/remote/ubuntu@smithi066.front.sepia.ceph.com/crash 2023-11-17T05:38:16.949 DEBUG:teuthology.orchestra.run.smithi066:> sudo tar c -f - -C /var/lib/ceph/ed6470e2-8508-11ee-8dbe-212e2dc638e7/crash -- . 2023-11-17T05:38:16.992 INFO:teuthology.orchestra.run.smithi066.stderr:tar: /var/lib/ceph/ed6470e2-8508-11ee-8dbe-212e2dc638e7/crash: Cannot open: No such file or directory 2023-11-17T05:38:16.992 INFO:teuthology.orchestra.run.smithi066.stderr:tar: Error is not recoverable: exiting now 2023-11-17T05:38:16.993 INFO:tasks.cephadm:Checking cluster log for badness... 2023-11-17T05:38:16.993 DEBUG:teuthology.orchestra.run.smithi040:> sudo egrep '\[ERR\]|\[WRN\]|\[SEC\]' /var/log/ceph/ed6470e2-8508-11ee-8dbe-212e2dc638e7/ceph.log | egrep -v '\(MDS_ALL_DOWN\)' | egrep -v '\(MDS_UP_LESS_THAN_MAX\)' | egrep -v 'overall HEALTH_' | egrep -v '\(FS_DEGRADED\)' | egrep -v '\(MDS_FAILED\)' | egrep -v '\(MDS_DEGRADED\)' | egrep -v '\(FS_WITH_FAILED_MDS\)' | egrep -v '\(MDS_DAMAGE\)' | egrep -v '\(MDS_ALL_DOWN\)' | egrep -v '\(MDS_UP_LESS_THAN_MAX\)' | egrep -v '\(FS_INLINE_DATA_DEPRECATED\)' | egrep -v 'overall HEALTH_' | egrep -v '\(OSD_DOWN\)' | egrep -v '\(OSD_' | egrep -v 'but it is still running' | egrep -v 'is not responding' | head -n 1 2023-11-17T05:38:17.040 INFO:teuthology.orchestra.run.smithi040.stderr:grep: /var/log/ceph/ed6470e2-8508-11ee-8dbe-212e2dc638e7/ceph.log: No such file or directory 2023-11-17T05:38:17.040 INFO:tasks.cephadm:Compressing logs... 2023-11-17T05:38:17.040 DEBUG:teuthology.orchestra.run.smithi040:> time sudo find /var/log/ceph /var/log/rbd-target-api -name '*.log' -print0 | sudo xargs --max-args=1 --max-procs=0 --verbose -0 --no-run-if-empty -- gzip -5 --verbose -- 2023-11-17T05:38:17.083 DEBUG:teuthology.orchestra.run.smithi066:> time sudo find /var/log/ceph /var/log/rbd-target-api -name '*.log' -print0 | sudo xargs --max-args=1 --max-procs=0 --verbose -0 --no-run-if-empty -- gzip -5 --verbose -- 2023-11-17T05:38:17.111 INFO:teuthology.orchestra.run.smithi066.stderr:find: ‘/var/log/rbd-target-api’gzip -5 --verbose -- /var/log/ceph/cephadm.log 2023-11-17T05:38:17.111 INFO:teuthology.orchestra.run.smithi066.stderr:: No such file or directory 2023-11-17T05:38:17.111 INFO:teuthology.orchestra.run.smithi066.stderr:gzip -5 --verbose -- /var/log/ceph/ed6470e2-8508-11ee-8dbe-212e2dc638e7/ceph-volume.log 2023-11-17T05:38:17.111 INFO:teuthology.orchestra.run.smithi066.stderr:/var/log/ceph/cephadm.log: gzip -5 --verbose -- /var/log/ceph/ed6470e2-8508-11ee-8dbe-212e2dc638e7/ceph-mgr.smithi066.aazujw.log 2023-11-17T05:38:17.111 INFO:teuthology.orchestra.run.smithi066.stderr:/var/log/ceph/ed6470e2-8508-11ee-8dbe-212e2dc638e7/ceph-volume.log: gzip -5 --verbose -- /var/log/ceph/ed6470e2-8508-11ee-8dbe-212e2dc638e7/ceph-mon.smithi066.log 2023-11-17T05:38:17.112 INFO:teuthology.orchestra.run.smithi066.stderr:/var/log/ceph/ed6470e2-8508-11ee-8dbe-212e2dc638e7/ceph-mgr.smithi066.aazujw.log: 88.3% -- replaced with /var/log/ceph/ed6470e2-8508-11ee-8dbe-212e2dc638e7/ceph-volume.log.gz 2023-11-17T05:38:17.114 INFO:teuthology.orchestra.run.smithi066.stderr:/var/log/ceph/ed6470e2-8508-11ee-8dbe-212e2dc638e7/ceph-mon.smithi066.log: 91.2% -- replaced with /var/log/ceph/ed6470e2-8508-11ee-8dbe-212e2dc638e7/ceph-mgr.smithi066.aazujw.log.gz 2023-11-17T05:38:17.115 INFO:teuthology.orchestra.run.smithi040.stderr:find: ‘/var/log/rbd-target-api’: No such file or directory 2023-11-17T05:38:17.115 INFO:teuthology.orchestra.run.smithi040.stderr:gzip -5 --verbose -- /var/log/ceph/cephadm.log 2023-11-17T05:38:17.116 INFO:teuthology.orchestra.run.smithi040.stderr:gzip -5 --verbose -- /var/log/ceph/ed6470e2-8508-11ee-8dbe-212e2dc638e7/ceph-mgr.smithi040.ogcprl.log 2023-11-17T05:38:17.116 INFO:teuthology.orchestra.run.smithi040.stderr:gzip -5 --verbose -- /var/log/ceph/ed6470e2-8508-11ee-8dbe-212e2dc638e7/ceph-volume.log 2023-11-17T05:38:17.117 INFO:teuthology.orchestra.run.smithi066.stderr: 95.7% -- replaced with /var/log/ceph/cephadm.log.gz 2023-11-17T05:38:17.117 INFO:teuthology.orchestra.run.smithi066.stderr: 92.4% -- replaced with /var/log/ceph/ed6470e2-8508-11ee-8dbe-212e2dc638e7/ceph-mon.smithi066.log.gz 2023-11-17T05:38:17.117 INFO:teuthology.orchestra.run.smithi040.stderr:/var/log/ceph/cephadm.log: gzip -5 --verbose -- /var/log/ceph/ed6470e2-8508-11ee-8dbe-212e2dc638e7/ceph-osd.0.log 2023-11-17T05:38:17.117 INFO:teuthology.orchestra.run.smithi040.stderr:/var/log/ceph/ed6470e2-8508-11ee-8dbe-212e2dc638e7/ceph-mgr.smithi040.ogcprl.log: gzip -5 --verbose -- /var/log/ceph/ed6470e2-8508-11ee-8dbe-212e2dc638e7/ceph-mon.smithi040.log 2023-11-17T05:38:17.118 INFO:teuthology.orchestra.run.smithi066.stderr: 2023-11-17T05:38:17.118 INFO:teuthology.orchestra.run.smithi066.stderr:real 0m0.018s 2023-11-17T05:38:17.118 INFO:teuthology.orchestra.run.smithi066.stderr:user 0m0.026s 2023-11-17T05:38:17.118 INFO:teuthology.orchestra.run.smithi066.stderr:sys 0m0.010s 2023-11-17T05:38:17.119 INFO:teuthology.orchestra.run.smithi040.stderr:/var/log/ceph/ed6470e2-8508-11ee-8dbe-212e2dc638e7/ceph-volume.log: /var/log/ceph/ed6470e2-8508-11ee-8dbe-212e2dc638e7/ceph-osd.0.log: /var/log/ceph/ed6470e2-8508-11ee-8dbe-212e2dc638e7/ceph-mon.smithi040.log: 89.0% -- replaced with /var/log/ceph/ed6470e2-8508-11ee-8dbe-212e2dc638e7/ceph-volume.log.gz 2023-11-17T05:38:17.119 INFO:teuthology.orchestra.run.smithi040.stderr: 91.4% -- replaced with /var/log/ceph/ed6470e2-8508-11ee-8dbe-212e2dc638e7/ceph-osd.0.log.gz 2023-11-17T05:38:17.125 INFO:teuthology.orchestra.run.smithi040.stderr: 91.5% -- replaced with /var/log/ceph/cephadm.log.gz 2023-11-17T05:38:17.246 INFO:teuthology.orchestra.run.smithi040.stderr: 93.0% -- replaced with /var/log/ceph/ed6470e2-8508-11ee-8dbe-212e2dc638e7/ceph-mgr.smithi040.ogcprl.log.gz 2023-11-17T05:38:17.341 INFO:teuthology.orchestra.run.smithi040.stderr: 90.9% -- replaced with /var/log/ceph/ed6470e2-8508-11ee-8dbe-212e2dc638e7/ceph-mon.smithi040.log.gz 2023-11-17T05:38:17.342 INFO:teuthology.orchestra.run.smithi040.stderr: 2023-11-17T05:38:17.343 INFO:teuthology.orchestra.run.smithi040.stderr:real 0m0.240s 2023-11-17T05:38:17.343 INFO:teuthology.orchestra.run.smithi040.stderr:user 0m0.364s 2023-11-17T05:38:17.343 INFO:teuthology.orchestra.run.smithi040.stderr:sys 0m0.030s 2023-11-17T05:38:17.343 INFO:tasks.cephadm:Archiving logs... 2023-11-17T05:38:17.343 DEBUG:teuthology.misc:Transferring archived files from smithi040:/var/log/ceph to /home/teuthworker/archive/yuriw-2023-11-16_22:29:26-rados-wip-yuri3-testing-2023-11-14-1227-pacific-distro-default-smithi/7461274/remote/ubuntu@smithi040.front.sepia.ceph.com/log 2023-11-17T05:38:17.344 DEBUG:teuthology.orchestra.run.smithi040:> sudo tar c -f - -C /var/log/ceph -- . 2023-11-17T05:38:17.539 DEBUG:teuthology.misc:Transferring archived files from smithi066:/var/log/ceph to /home/teuthworker/archive/yuriw-2023-11-16_22:29:26-rados-wip-yuri3-testing-2023-11-14-1227-pacific-distro-default-smithi/7461274/remote/ubuntu@smithi066.front.sepia.ceph.com/log 2023-11-17T05:38:17.540 DEBUG:teuthology.orchestra.run.smithi066:> sudo tar c -f - -C /var/log/ceph -- . 2023-11-17T05:38:17.575 INFO:tasks.cephadm:Removing cluster... 2023-11-17T05:38:17.575 DEBUG:teuthology.orchestra.run.smithi040:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid ed6470e2-8508-11ee-8dbe-212e2dc638e7 --force 2023-11-17T05:38:17.917 DEBUG:teuthology.orchestra.run.smithi066:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid ed6470e2-8508-11ee-8dbe-212e2dc638e7 --force 2023-11-17T05:38:18.232 INFO:tasks.cephadm:Removing cephadm ... 2023-11-17T05:38:18.232 DEBUG:teuthology.orchestra.run.smithi040:> rm -rf /home/ubuntu/cephtest/cephadm 2023-11-17T05:38:18.255 DEBUG:teuthology.orchestra.run.smithi066:> rm -rf /home/ubuntu/cephtest/cephadm 2023-11-17T05:38:18.273 INFO:tasks.cephadm:Teardown complete 2023-11-17T05:38:18.274 ERROR:teuthology.run_tasks:Saw exception from tasks. Traceback (most recent call last): File "/home/teuthworker/src/git.ceph.com_teuthology_6899cd26fceddb2fec83dc1a1349394b28c8998e/teuthology/run_tasks.py", line 109, in run_tasks manager.__enter__() File "/usr/lib/python3.8/contextlib.py", line 113, in __enter__ return next(self.gen) File "/home/teuthworker/src/github.com_ceph_ceph-c_ed3c8e926b9de93026c8a7c1d86be5d6821b52ac/qa/tasks/cephadm.py", line 1463, in task with contextutil.nested( File "/usr/lib/python3.8/contextlib.py", line 113, in __enter__ return next(self.gen) File "/home/teuthworker/src/git.ceph.com_teuthology_6899cd26fceddb2fec83dc1a1349394b28c8998e/teuthology/contextutil.py", line 54, in nested raise exc[1] File "/usr/lib/python3.8/contextlib.py", line 131, in __exit__ self.gen.throw(type, value, traceback) File "/home/teuthworker/src/github.com_ceph_ceph-c_ed3c8e926b9de93026c8a7c1d86be5d6821b52ac/qa/tasks/cephadm.py", line 1376, in initialize_config yield File "/home/teuthworker/src/git.ceph.com_teuthology_6899cd26fceddb2fec83dc1a1349394b28c8998e/teuthology/contextutil.py", line 30, in nested vars.append(enter()) File "/usr/lib/python3.8/contextlib.py", line 113, in __enter__ return next(self.gen) File "/home/teuthworker/src/github.com_ceph_ceph-c_ed3c8e926b9de93026c8a7c1d86be5d6821b52ac/qa/tasks/cephadm.py", line 752, in ceph_osds _shell(ctx, cluster_name, remote, [ File "/home/teuthworker/src/github.com_ceph_ceph-c_ed3c8e926b9de93026c8a7c1d86be5d6821b52ac/qa/tasks/cephadm.py", line 34, in _shell return remote.run( File "/home/teuthworker/src/git.ceph.com_teuthology_6899cd26fceddb2fec83dc1a1349394b28c8998e/teuthology/orchestra/remote.py", line 522, in run r = self._runner(client=self.ssh, name=self.shortname, **kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_6899cd26fceddb2fec83dc1a1349394b28c8998e/teuthology/orchestra/run.py", line 455, in run r.wait() File "/home/teuthworker/src/git.ceph.com_teuthology_6899cd26fceddb2fec83dc1a1349394b28c8998e/teuthology/orchestra/run.py", line 161, in wait self._raise_for_status() File "/home/teuthworker/src/git.ceph.com_teuthology_6899cd26fceddb2fec83dc1a1349394b28c8998e/teuthology/orchestra/run.py", line 181, in _raise_for_status raise CommandFailedError( teuthology.exceptions.CommandFailedError: Command failed on smithi040 with status 22: 'sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.4 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ed6470e2-8508-11ee-8dbe-212e2dc638e7 -- ceph orch daemon add osd smithi040:vg_nvme/lv_4' 2023-11-17T05:38:18.346 ERROR:teuthology.util.sentry: Sentry event: https://sentry.ceph.com/organizations/ceph/?query=e6b58c43d72b4bdbbf973fc60a13fb1d Traceback (most recent call last): File "/home/teuthworker/src/git.ceph.com_teuthology_6899cd26fceddb2fec83dc1a1349394b28c8998e/teuthology/run_tasks.py", line 109, in run_tasks manager.__enter__() File "/usr/lib/python3.8/contextlib.py", line 113, in __enter__ return next(self.gen) File "/home/teuthworker/src/github.com_ceph_ceph-c_ed3c8e926b9de93026c8a7c1d86be5d6821b52ac/qa/tasks/cephadm.py", line 1463, in task with contextutil.nested( File "/usr/lib/python3.8/contextlib.py", line 113, in __enter__ return next(self.gen) File "/home/teuthworker/src/git.ceph.com_teuthology_6899cd26fceddb2fec83dc1a1349394b28c8998e/teuthology/contextutil.py", line 54, in nested raise exc[1] File "/usr/lib/python3.8/contextlib.py", line 131, in __exit__ self.gen.throw(type, value, traceback) File "/home/teuthworker/src/github.com_ceph_ceph-c_ed3c8e926b9de93026c8a7c1d86be5d6821b52ac/qa/tasks/cephadm.py", line 1376, in initialize_config yield File "/home/teuthworker/src/git.ceph.com_teuthology_6899cd26fceddb2fec83dc1a1349394b28c8998e/teuthology/contextutil.py", line 30, in nested vars.append(enter()) File "/usr/lib/python3.8/contextlib.py", line 113, in __enter__ return next(self.gen) File "/home/teuthworker/src/github.com_ceph_ceph-c_ed3c8e926b9de93026c8a7c1d86be5d6821b52ac/qa/tasks/cephadm.py", line 752, in ceph_osds _shell(ctx, cluster_name, remote, [ File "/home/teuthworker/src/github.com_ceph_ceph-c_ed3c8e926b9de93026c8a7c1d86be5d6821b52ac/qa/tasks/cephadm.py", line 34, in _shell return remote.run( File "/home/teuthworker/src/git.ceph.com_teuthology_6899cd26fceddb2fec83dc1a1349394b28c8998e/teuthology/orchestra/remote.py", line 522, in run r = self._runner(client=self.ssh, name=self.shortname, **kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_6899cd26fceddb2fec83dc1a1349394b28c8998e/teuthology/orchestra/run.py", line 455, in run r.wait() File "/home/teuthworker/src/git.ceph.com_teuthology_6899cd26fceddb2fec83dc1a1349394b28c8998e/teuthology/orchestra/run.py", line 161, in wait self._raise_for_status() File "/home/teuthworker/src/git.ceph.com_teuthology_6899cd26fceddb2fec83dc1a1349394b28c8998e/teuthology/orchestra/run.py", line 181, in _raise_for_status raise CommandFailedError( teuthology.exceptions.CommandFailedError: Command failed on smithi040 with status 22: 'sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.4 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ed6470e2-8508-11ee-8dbe-212e2dc638e7 -- ceph orch daemon add osd smithi040:vg_nvme/lv_4' 2023-11-17T05:38:18.349 DEBUG:teuthology.run_tasks:Unwinding manager cephadm 2023-11-17T05:38:18.362 DEBUG:teuthology.run_tasks:Unwinding manager install 2023-11-17T05:38:18.391 INFO:teuthology.task.install.util:Removing shipped files: /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits /usr/bin/stdin-killer... 2023-11-17T05:38:18.391 DEBUG:teuthology.orchestra.run.smithi040:> sudo rm -f -- /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits /usr/bin/stdin-killer 2023-11-17T05:38:18.411 DEBUG:teuthology.orchestra.run.smithi066:> sudo rm -f -- /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits /usr/bin/stdin-killer 2023-11-17T05:38:18.507 DEBUG:teuthology.run_tasks:Unwinding manager clock 2023-11-17T05:38:18.517 INFO:teuthology.task.clock:Checking final clock skew... 2023-11-17T05:38:18.518 DEBUG:teuthology.orchestra.run.smithi040:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2023-11-17T05:38:18.534 DEBUG:teuthology.orchestra.run.smithi066:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2023-11-17T05:38:18.546 INFO:teuthology.orchestra.run.smithi040.stderr:bash: ntpq: command not found 2023-11-17T05:38:18.548 INFO:teuthology.orchestra.run.smithi040.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2023-11-17T05:38:18.548 INFO:teuthology.orchestra.run.smithi040.stdout:=============================================================================== 2023-11-17T05:38:18.548 INFO:teuthology.orchestra.run.smithi040.stdout:^+ hv01.front.sepia.ceph.com 2 6 377 4 +551us[ +551us] +/- 70ms 2023-11-17T05:38:18.548 INFO:teuthology.orchestra.run.smithi040.stdout:^* hv02.front.sepia.ceph.com 3 6 377 4 -511us[ -506us] +/- 51ms 2023-11-17T05:38:18.548 INFO:teuthology.orchestra.run.smithi040.stdout:^+ hv03.front.sepia.ceph.com 3 6 377 2 +316us[ +316us] +/- 64ms 2023-11-17T05:38:18.549 INFO:teuthology.orchestra.run.smithi040.stdout:^? hv04.front.sepia.ceph.com 0 9 0 - +0ns[ +0ns] +/- 0ns 2023-11-17T05:38:18.553 INFO:teuthology.orchestra.run.smithi066.stderr:bash: ntpq: command not found 2023-11-17T05:38:18.558 INFO:teuthology.orchestra.run.smithi066.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2023-11-17T05:38:18.559 INFO:teuthology.orchestra.run.smithi066.stdout:=============================================================================== 2023-11-17T05:38:18.559 INFO:teuthology.orchestra.run.smithi066.stdout:^+ hv01.front.sepia.ceph.com 2 6 377 4 +547us[ +547us] +/- 70ms 2023-11-17T05:38:18.559 INFO:teuthology.orchestra.run.smithi066.stdout:^* hv02.front.sepia.ceph.com 3 6 377 5 -513us[ -516us] +/- 51ms 2023-11-17T05:38:18.560 INFO:teuthology.orchestra.run.smithi066.stdout:^+ hv03.front.sepia.ceph.com 3 6 377 6 +302us[ +299us] +/- 64ms 2023-11-17T05:38:18.560 INFO:teuthology.orchestra.run.smithi066.stdout:^? hv04.front.sepia.ceph.com 0 9 0 - +0ns[ +0ns] +/- 0ns 2023-11-17T05:38:18.560 DEBUG:teuthology.run_tasks:Unwinding manager ansible.cephlab 2023-11-17T05:38:18.576 INFO:teuthology.task.ansible:Skipping ansible cleanup... 2023-11-17T05:38:18.577 DEBUG:teuthology.run_tasks:Unwinding manager selinux 2023-11-17T05:38:18.596 DEBUG:teuthology.orchestra.run.smithi040:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2023-11-17T05:38:18.699 DEBUG:teuthology.orchestra.run.smithi066:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2023-11-17T05:38:18.795 DEBUG:teuthology.orchestra.run.smithi040:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 2023-11-17T05:38:18.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-11-17T05:38:18.836 DEBUG:teuthology.orchestra.run.smithi066:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 2023-11-17T05:38:18.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-11-17T05:38:18.868 DEBUG:teuthology.run_tasks:Unwinding manager pcp 2023-11-17T05:38:18.878 DEBUG:teuthology.run_tasks:Unwinding manager internal.timer 2023-11-17T05:38:18.920 INFO:teuthology.task.internal:Duration was 2059.346339 seconds 2023-11-17T05:38:18.920 DEBUG:teuthology.run_tasks:Unwinding manager internal.syslog 2023-11-17T05:38:18.943 INFO:teuthology.task.internal.syslog:Shutting down syslog monitoring... 2023-11-17T05:38:18.943 DEBUG:teuthology.orchestra.run.smithi040:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2023-11-17T05:38:18.947 DEBUG:teuthology.orchestra.run.smithi066:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2023-11-17T05:38:18.993 INFO:teuthology.orchestra.run.smithi066.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2023-11-17T05:38:19.002 INFO:teuthology.orchestra.run.smithi040.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2023-11-17T05:38:19.332 INFO:teuthology.task.internal.syslog:Checking logs for errors... 2023-11-17T05:38:19.332 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi040.front.sepia.ceph.com 2023-11-17T05:38:19.332 DEBUG:teuthology.orchestra.run.smithi040:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2023-11-17T05:38:19.362 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi066.front.sepia.ceph.com 2023-11-17T05:38:19.362 DEBUG:teuthology.orchestra.run.smithi066:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2023-11-17T05:38:19.386 INFO:teuthology.task.internal.syslog:Compressing syslogs... 2023-11-17T05:38:19.386 DEBUG:teuthology.orchestra.run.smithi040:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2023-11-17T05:38:19.405 DEBUG:teuthology.orchestra.run.smithi066:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2023-11-17T05:38:19.475 INFO:teuthology.task.internal.syslog:Gathering journactl -b0... 2023-11-17T05:38:19.475 DEBUG:teuthology.orchestra.run.smithi040:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2023-11-17T05:38:19.478 DEBUG:teuthology.orchestra.run.smithi066:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2023-11-17T05:38:19.574 DEBUG:teuthology.run_tasks:Unwinding manager internal.sudo 2023-11-17T05:38:19.586 INFO:teuthology.task.internal:Restoring /etc/sudoers... 2023-11-17T05:38:19.586 DEBUG:teuthology.orchestra.run.smithi040:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2023-11-17T05:38:19.617 DEBUG:teuthology.orchestra.run.smithi066:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2023-11-17T05:38:19.646 DEBUG:teuthology.run_tasks:Unwinding manager internal.coredump 2023-11-17T05:38:19.659 DEBUG:teuthology.orchestra.run.smithi040:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2023-11-17T05:38:19.662 DEBUG:teuthology.orchestra.run.smithi066:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2023-11-17T05:38:19.692 INFO:teuthology.orchestra.run.smithi040.stdout:kernel.core_pattern = core 2023-11-17T05:38:19.713 INFO:teuthology.orchestra.run.smithi066.stdout:kernel.core_pattern = core 2023-11-17T05:38:19.738 DEBUG:teuthology.orchestra.run.smithi040:> test -e /home/ubuntu/cephtest/archive/coredump 2023-11-17T05:38:19.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-11-17T05:38:19.796 DEBUG:teuthology.orchestra.run.smithi066:> test -e /home/ubuntu/cephtest/archive/coredump 2023-11-17T05:38:19.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-11-17T05:38:19.815 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive 2023-11-17T05:38:19.826 INFO:teuthology.task.internal:Transferring archived files... 2023-11-17T05:38:19.828 DEBUG:teuthology.misc:Transferring archived files from smithi040:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/yuriw-2023-11-16_22:29:26-rados-wip-yuri3-testing-2023-11-14-1227-pacific-distro-default-smithi/7461274/remote/smithi040 2023-11-17T05:38:19.829 DEBUG:teuthology.orchestra.run.smithi040:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2023-11-17T05:38:19.896 DEBUG:teuthology.misc:Transferring archived files from smithi066:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/yuriw-2023-11-16_22:29:26-rados-wip-yuri3-testing-2023-11-14-1227-pacific-distro-default-smithi/7461274/remote/smithi066 2023-11-17T05:38:19.914 DEBUG:teuthology.orchestra.run.smithi066:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2023-11-17T05:38:19.958 INFO:teuthology.task.internal:Removing archive directory... 2023-11-17T05:38:19.958 DEBUG:teuthology.orchestra.run.smithi040:> rm -rf -- /home/ubuntu/cephtest/archive 2023-11-17T05:38:19.963 DEBUG:teuthology.orchestra.run.smithi066:> rm -rf -- /home/ubuntu/cephtest/archive 2023-11-17T05:38:20.008 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive_upload 2023-11-17T05:38:20.022 INFO:teuthology.task.internal:Not uploading archives. 2023-11-17T05:38:20.022 DEBUG:teuthology.run_tasks:Unwinding manager internal.base 2023-11-17T05:38:20.069 INFO:teuthology.task.internal:Tidying up after the test... 2023-11-17T05:38:20.069 DEBUG:teuthology.orchestra.run.smithi040:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2023-11-17T05:38:20.071 DEBUG:teuthology.orchestra.run.smithi066:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2023-11-17T05:38:20.089 INFO:teuthology.orchestra.run.smithi066.stdout: 393224 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Nov 17 05:38 /home/ubuntu/cephtest 2023-11-17T05:38:20.096 INFO:teuthology.orchestra.run.smithi040.stdout: 393224 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Nov 17 05:38 /home/ubuntu/cephtest 2023-11-17T05:38:20.096 DEBUG:teuthology.run_tasks:Unwinding manager kernel 2023-11-17T05:38:20.113 DEBUG:teuthology.run_tasks:Unwinding manager console_log 2023-11-17T05:38:20.151 INFO:teuthology.nuke:Checking targets against current locks 2023-11-17T05:38:20.178 DEBUG:teuthology.nuke:shortname: smithi040 2023-11-17T05:38:20.178 INFO:teuthology.task.internal.check_lock:Checking locks... 2023-11-17T05:38:20.203 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi040.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/yuriw-2023-11-16_22:29:26-rados-wip-yuri3-testing-2023-11-14-1227-pacific-distro-default-smithi/7461274', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '8', 'arch': 'x86_64', 'locked': True, 'locked_since': '2023-11-17 04:52:24.161988', 'locked_by': 'scheduled_yuriw@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBKtHuG7UjUvnBw2JtooGL3kFkXlILUs0Oz0JYpqOe5VKp5gQo30S3BAyRJbfQGXgp1jj8Z4Ql6GUm7/BV4/gpLo='} 2023-11-17T05:38:20.206 DEBUG:teuthology.nuke:shortname: smithi066 2023-11-17T05:38:20.207 INFO:teuthology.task.internal.check_lock:Checking locks... 2023-11-17T05:38:20.233 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi066.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/yuriw-2023-11-16_22:29:26-rados-wip-yuri3-testing-2023-11-14-1227-pacific-distro-default-smithi/7461274', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '8', 'arch': 'x86_64', 'locked': True, 'locked_since': '2023-11-17 04:52:24.163889', 'locked_by': 'scheduled_yuriw@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBLuHCVetiuoY465BMAkkaNSxMexMeQjvFwnGnetv25pgKW80KHb0O0XeMxOl8GRkSeP084DKy25hDE6z+CucA4E='} 2023-11-17T05:38:20.276 INFO:teuthology.orchestra.console.smithi040:Power off 2023-11-17T05:38:20.276 DEBUG:teuthology.orchestra.console.smithi040:pexpect command: ipmitool -H smithi040.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power off 2023-11-17T05:38:20.308 DEBUG:teuthology.orchestra.console.smithi040:power off output: Chassis Power Control: Down/Off 2023-11-17T05:38:20.308 DEBUG:teuthology.orchestra.console.smithi040:pexpect command: ipmitool -H smithi040.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2023-11-17T05:38:20.337 INFO:teuthology.orchestra.console.smithi066:Power off 2023-11-17T05:38:20.337 DEBUG:teuthology.orchestra.console.smithi066:pexpect command: ipmitool -H smithi066.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power off 2023-11-17T05:38:20.357 DEBUG:teuthology.orchestra.console.smithi040:check power output: Chassis Power is on 2023-11-17T05:38:20.380 DEBUG:teuthology.orchestra.console.smithi066:power off output: Chassis Power Control: Down/Off 2023-11-17T05:38:20.381 DEBUG:teuthology.orchestra.console.smithi066:pexpect command: ipmitool -H smithi066.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2023-11-17T05:38:20.410 DEBUG:teuthology.orchestra.console.smithi066:check power output: Chassis Power is on 2023-11-17T05:38:24.358 DEBUG:teuthology.orchestra.console.smithi040:pexpect command: ipmitool -H smithi040.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2023-11-17T05:38:24.411 DEBUG:teuthology.orchestra.console.smithi066:pexpect command: ipmitool -H smithi066.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2023-11-17T05:38:24.469 DEBUG:teuthology.orchestra.console.smithi040:check power output: Chassis Power is on 2023-11-17T05:38:24.522 DEBUG:teuthology.orchestra.console.smithi066:check power output: Chassis Power is on 2023-11-17T05:38:28.474 DEBUG:teuthology.orchestra.console.smithi040:pexpect command: ipmitool -H smithi040.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2023-11-17T05:38:56.883 DEBUG:teuthology.orchestra.console.smithi066:pexpect command: ipmitool -H smithi066.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2023-11-17T05:38:56.985 DEBUG:teuthology.orchestra.console.smithi040:check power output: Chassis Power is off 2023-11-17T05:38:57.000 DEBUG:teuthology.orchestra.console.smithi066:check power output: Chassis Power is off 2023-11-17T05:38:57.087 INFO:teuthology.orchestra.console.smithi040:Power off completed 2023-11-17T05:38:57.101 INFO:teuthology.orchestra.console.smithi066:Power off completed 2023-11-17T05:38:57.208 INFO:teuthology.run:Summary data: description: rados/cephadm/mds_upgrade_sequence/{bluestore-bitmap centos_8.stream_container_tools conf/{client mds mon osd} overrides/{ignorelist_health ignorelist_wrongly_marked_down pg-warn} roles tasks/{0-from/v16.2.4 1-volume/{0-create 1-ranks/2 2-allow_standby_replay/no 3-inline/yes 4-verify} 2-client 3-upgrade-with-workload 4-verify}} duration: 2059.3463389873505 failure_reason: 'Command failed on smithi040 with status 22: ''sudo /home/ubuntu/cephtest/cephadm --image docker.io/ceph/ceph:v16.2.4 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ed6470e2-8508-11ee-8dbe-212e2dc638e7 -- ceph orch daemon add osd smithi040:vg_nvme/lv_4''' flavor: default owner: scheduled_yuriw@teuthology sentry_event: https://sentry.ceph.com/organizations/ceph/?query=e6b58c43d72b4bdbbf973fc60a13fb1d status: fail success: false 2023-11-17T05:38:57.209 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2023-11-17T05:38:57.305 INFO:teuthology.run:FAIL