2024-04-24T12:56:01.361 INFO:root:teuthology version: 0.0.1.dev269+g9481b1d 2024-04-24T12:56:01.362 DEBUG:teuthology.run:Teuthology command: teuthology --name adking-2024-04-24_11:41:41-orch:cephadm-wip-adk-testing-2024-04-23-1222-distro-default-smithi --description orch:cephadm/mgr-nfs-upgrade/{0-centos_9.stream 1-bootstrap/17.2.0 1-start 2-nfs 3-upgrade-with-workload 4-final} --verbose --owner scheduled_adking@teuthology --archive /home/teuthworker/archive/adking-2024-04-24_11:41:41-orch:cephadm-wip-adk-testing-2024-04-23-1222-distro-default-smithi/7671319 -- /home/teuthworker/archive/adking-2024-04-24_11:41:41-orch:cephadm-wip-adk-testing-2024-04-23-1222-distro-default-smithi/7671319/orig.config.yaml 2024-04-24T12:56:01.399 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2024-04-24T12:56:01.484 INFO:teuthology.run:Config: archive_path: /home/teuthworker/archive/adking-2024-04-24_11:41:41-orch:cephadm-wip-adk-testing-2024-04-23-1222-distro-default-smithi/7671319 branch: wip-adk-testing-2024-04-23-1222 description: orch:cephadm/mgr-nfs-upgrade/{0-centos_9.stream 1-bootstrap/17.2.0 1-start 2-nfs 3-upgrade-with-workload 4-final} email: adking@redhat.com first_in_suite: false job_id: '7671319' kernel: kdb: 1 sha1: distro last_in_suite: false machine_type: smithi name: adking-2024-04-24_11:41:41-orch:cephadm-wip-adk-testing-2024-04-23-1222-distro-default-smithi no_nested_subset: false nuke-on-error: true openstack: - volumes: count: 4 size: 10 os_type: centos os_version: 9.stream overrides: admin_socket: branch: wip-adk-testing-2024-04-23-1222 ceph: conf: mgr: debug mgr: 20 debug ms: 1 mon: debug mon: 20 debug ms: 1 debug paxos: 20 osd: debug ms: 1 debug osd: 20 osd shutdown pgref assert: true flavor: default log-ignorelist: - \(MDS_ALL_DOWN\) - \(MDS_UP_LESS_THAN_MAX\) log-only-match: - CEPHADM_ sha1: 23fcfb96e7e1a49d12a94e3f87a8e3f06db2a1ec ceph-deploy: conf: client: log file: /var/log/ceph/ceph-$name.$pid.log mon: {} install: ceph: flavor: default sha1: 23fcfb96e7e1a49d12a94e3f87a8e3f06db2a1ec selinux: allowlist: - scontext=system_u:system_r:logrotate_t:s0 - scontext=system_u:system_r:getty_t:s0 workunit: branch: wip-adk-testing-2024-04-23-1222 sha1: 23fcfb96e7e1a49d12a94e3f87a8e3f06db2a1ec owner: scheduled_adking@teuthology priority: 80 repo: https://git.ceph.com/ceph-ci.git roles: - - host.a - osd.0 - osd.1 - osd.2 - osd.3 - client.0 - - host.b - osd.4 - osd.5 - osd.6 - osd.7 seed: 6222 sha1: 23fcfb96e7e1a49d12a94e3f87a8e3f06db2a1ec sleep_before_teardown: 0 subset: 1/16 suite: orch:cephadm suite_branch: wip-adk-testing-2024-04-23-1222 suite_path: /home/teuthworker/src/git.ceph.com_ceph-c_23fcfb96e7e1a49d12a94e3f87a8e3f06db2a1ec/qa suite_relpath: qa suite_repo: https://git.ceph.com/ceph-ci.git suite_sha1: 23fcfb96e7e1a49d12a94e3f87a8e3f06db2a1ec targets: smithi069.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBH/Hfuo3Uojpx7SKQktHR32R8HvKf9OuIXSZ8JIl9/E/4Bd6qqiOIUd9ftefIKFCMsiPPA2UdaPElPjjs5W5bS0= smithi114.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBELW/agFKO94t3difRrRkj0O1Nn8I7WShXt1NPx7P+DTz41jSKcynLn4XKES8hblVhzzf3zZ8BmrnMB53qZI7gU= tasks: - cephadm: cephadm_branch: v17.2.0 cephadm_git_url: https://github.com/ceph/ceph image: quay.io/ceph/ceph:v17.2.0 roleless: true - cephadm.shell: host.a: - ceph orch status - ceph orch ps - ceph orch ls - ceph orch host ls - ceph orch device ls - vip.exec: all-hosts: - systemctl stop nfs-server - cephadm.shell: host.a: - ceph fs volume create foofs - cephadm.wait_for_service: service: mds.foofs - cephadm.shell: host.a: - ceph nfs cluster create foo --placement=2 || ceph nfs cluster create cephfs foo --placement=2 - ceph nfs export create cephfs --fsname foofs --clusterid foo --binding /fake || ceph nfs export create cephfs --fsname foofs --cluster-id foo --pseudo-path /fake - while ! ceph orch ls | grep nfs | grep 2/2 ; do sleep 1 ; done - vip.exec: host.a: - mkdir /mnt/foo - while ! mount -t nfs $(hostname):/fake /mnt/foo -o sync ; do sleep 5 ; done - echo test > /mnt/foo/testfile - sync - parallel: - upgrade-tasks - workload-tasks - vip.exec: host.a: - umount /mnt/foo - cephadm.shell: host.a: - ceph nfs cluster ls | grep foo - ceph nfs export ls foo --detailed - rados -p .nfs --all ls - - cephadm.shell: host.a: - 'set -ex [[ `ceph config get mgr mgr/cephadm/migration_current` -gt 2 ]] ' teuthology: fragments_dropped: [] meta: {} postmerge: [] teuthology_branch: main teuthology_sha1: 9481b1d62f50e7d0a4f3dd83adf6945b08d5ff17 timestamp: 2024-04-24_11:41:41 tube: smithi upgrade-tasks: sequential: - cephadm.shell: env: - sha1 host.a: - ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force - ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force - ceph config set global log_to_journald false --force - ceph mgr module enable nfs --force - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 - cephadm.shell: env: - sha1 host.a: - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; ceph health detail ; sleep 30 ; done - ceph orch ps - ceph versions - echo "wait for servicemap items w/ changing names to refresh" - sleep 60 - ceph orch ps - ceph orch upgrade status - ceph health detail - ceph versions - ceph versions | jq -e '.overall | length == 1' - ceph versions | jq -e '.overall | keys' | grep $sha1 - cephadm.wait_for_service: service: nfs.foo user: adking verbose: false worker_log: /home/teuthworker/archive/worker_logs/dispatcher.smithi.2226885 workload-tasks: sequential: - exec: host.a: - cd /mnt/foo && dbench 5 -t 600 || true - umount /mnt/foo - while ! mount -t nfs $(hostname):/fake /mnt/foo ; do sleep 5 ; done - cd /mnt/foo && dbench 5 -t 5 2024-04-24T12:56:01.484 INFO:teuthology.run:suite_path is set to /home/teuthworker/src/git.ceph.com_ceph-c_23fcfb96e7e1a49d12a94e3f87a8e3f06db2a1ec/qa; will attempt to use it 2024-04-24T12:56:01.485 INFO:teuthology.run:Found tasks at /home/teuthworker/src/git.ceph.com_ceph-c_23fcfb96e7e1a49d12a94e3f87a8e3f06db2a1ec/qa/tasks 2024-04-24T12:56:01.485 INFO:teuthology.run_tasks:Running task internal.check_packages... 2024-04-24T12:56:01.487 INFO:teuthology.task.internal:Checking packages... 2024-04-24T12:56:01.513 INFO:teuthology.task.internal:Checking packages for os_type 'centos', flavor 'default' and ceph hash '23fcfb96e7e1a49d12a94e3f87a8e3f06db2a1ec' 2024-04-24T12:56:01.513 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-04-24T12:56:01.513 INFO:teuthology.packaging:ref: None 2024-04-24T12:56:01.513 INFO:teuthology.packaging:tag: None 2024-04-24T12:56:01.513 INFO:teuthology.packaging:branch: wip-adk-testing-2024-04-23-1222 2024-04-24T12:56:01.513 INFO:teuthology.packaging:sha1: 23fcfb96e7e1a49d12a94e3f87a8e3f06db2a1ec 2024-04-24T12:56:01.513 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&ref=wip-adk-testing-2024-04-23-1222 2024-04-24T12:56:01.656 INFO:teuthology.task.internal:Found packages for ceph version 19.0.0-3259.g23fcfb96 2024-04-24T12:56:01.659 INFO:teuthology.run_tasks:Running task internal.buildpackages_prep... 2024-04-24T12:56:01.666 INFO:teuthology.task.internal:no buildpackages task found 2024-04-24T12:56:01.666 INFO:teuthology.run_tasks:Running task internal.save_config... 2024-04-24T12:56:01.683 INFO:teuthology.task.internal:Saving configuration 2024-04-24T12:56:01.697 INFO:teuthology.run_tasks:Running task internal.check_lock... 2024-04-24T12:56:01.704 INFO:teuthology.task.internal.check_lock:Checking locks... 2024-04-24T12:56:01.726 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi069.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/adking-2024-04-24_11:41:41-orch:cephadm-wip-adk-testing-2024-04-23-1222-distro-default-smithi/7671319', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-04-24 12:50:59.575279', 'locked_by': 'scheduled_adking@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBH/Hfuo3Uojpx7SKQktHR32R8HvKf9OuIXSZ8JIl9/E/4Bd6qqiOIUd9ftefIKFCMsiPPA2UdaPElPjjs5W5bS0='} 2024-04-24T12:56:01.745 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi114.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/adking-2024-04-24_11:41:41-orch:cephadm-wip-adk-testing-2024-04-23-1222-distro-default-smithi/7671319', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-04-24 12:50:59.573290', 'locked_by': 'scheduled_adking@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBELW/agFKO94t3difRrRkj0O1Nn8I7WShXt1NPx7P+DTz41jSKcynLn4XKES8hblVhzzf3zZ8BmrnMB53qZI7gU='} 2024-04-24T12:56:01.746 INFO:teuthology.run_tasks:Running task internal.add_remotes... 2024-04-24T12:56:01.752 INFO:teuthology.task.internal:roles: ubuntu@smithi069.front.sepia.ceph.com - ['host.a', 'osd.0', 'osd.1', 'osd.2', 'osd.3', 'client.0'] 2024-04-24T12:56:01.752 INFO:teuthology.task.internal:roles: ubuntu@smithi114.front.sepia.ceph.com - ['host.b', 'osd.4', 'osd.5', 'osd.6', 'osd.7'] 2024-04-24T12:56:01.752 INFO:teuthology.run_tasks:Running task console_log... 2024-04-24T12:56:01.824 DEBUG:teuthology.exit:Installing handler: Handler(exiter=, func=.kill_console_loggers at 0x7f0101651160>, signals=[15]) 2024-04-24T12:56:01.825 INFO:teuthology.run_tasks:Running task internal.connect... 2024-04-24T12:56:01.833 INFO:teuthology.task.internal:Opening connections... 2024-04-24T12:56:01.833 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi069.front.sepia.ceph.com 2024-04-24T12:56:01.835 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi069.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-24T12:56:01.912 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi114.front.sepia.ceph.com 2024-04-24T12:56:01.913 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi114.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-24T12:56:01.986 INFO:teuthology.run_tasks:Running task internal.push_inventory... 2024-04-24T12:56:01.993 DEBUG:teuthology.orchestra.run.smithi069:> uname -m 2024-04-24T12:56:02.012 INFO:teuthology.orchestra.run.smithi069.stdout:x86_64 2024-04-24T12:56:02.012 DEBUG:teuthology.orchestra.run.smithi069:> cat /etc/os-release 2024-04-24T12:56:02.070 INFO:teuthology.orchestra.run.smithi069.stdout:NAME="CentOS Stream" 2024-04-24T12:56:02.071 INFO:teuthology.orchestra.run.smithi069.stdout:VERSION="9" 2024-04-24T12:56:02.071 INFO:teuthology.orchestra.run.smithi069.stdout:ID="centos" 2024-04-24T12:56:02.071 INFO:teuthology.orchestra.run.smithi069.stdout:ID_LIKE="rhel fedora" 2024-04-24T12:56:02.071 INFO:teuthology.orchestra.run.smithi069.stdout:VERSION_ID="9" 2024-04-24T12:56:02.071 INFO:teuthology.orchestra.run.smithi069.stdout:PLATFORM_ID="platform:el9" 2024-04-24T12:56:02.071 INFO:teuthology.orchestra.run.smithi069.stdout:PRETTY_NAME="CentOS Stream 9" 2024-04-24T12:56:02.071 INFO:teuthology.orchestra.run.smithi069.stdout:ANSI_COLOR="0;31" 2024-04-24T12:56:02.071 INFO:teuthology.orchestra.run.smithi069.stdout:LOGO="fedora-logo-icon" 2024-04-24T12:56:02.071 INFO:teuthology.orchestra.run.smithi069.stdout:CPE_NAME="cpe:/o:centos:centos:9" 2024-04-24T12:56:02.071 INFO:teuthology.orchestra.run.smithi069.stdout:HOME_URL="https://centos.org/" 2024-04-24T12:56:02.071 INFO:teuthology.orchestra.run.smithi069.stdout:BUG_REPORT_URL="https://bugzilla.redhat.com/" 2024-04-24T12:56:02.071 INFO:teuthology.orchestra.run.smithi069.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 9" 2024-04-24T12:56:02.071 INFO:teuthology.orchestra.run.smithi069.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream" 2024-04-24T12:56:02.072 INFO:teuthology.lock.ops:Updating smithi069.front.sepia.ceph.com on lock server 2024-04-24T12:56:02.093 DEBUG:teuthology.orchestra.run.smithi114:> uname -m 2024-04-24T12:56:02.111 INFO:teuthology.orchestra.run.smithi114.stdout:x86_64 2024-04-24T12:56:02.112 DEBUG:teuthology.orchestra.run.smithi114:> cat /etc/os-release 2024-04-24T12:56:02.173 INFO:teuthology.orchestra.run.smithi114.stdout:NAME="CentOS Stream" 2024-04-24T12:56:02.173 INFO:teuthology.orchestra.run.smithi114.stdout:VERSION="9" 2024-04-24T12:56:02.173 INFO:teuthology.orchestra.run.smithi114.stdout:ID="centos" 2024-04-24T12:56:02.173 INFO:teuthology.orchestra.run.smithi114.stdout:ID_LIKE="rhel fedora" 2024-04-24T12:56:02.173 INFO:teuthology.orchestra.run.smithi114.stdout:VERSION_ID="9" 2024-04-24T12:56:02.173 INFO:teuthology.orchestra.run.smithi114.stdout:PLATFORM_ID="platform:el9" 2024-04-24T12:56:02.174 INFO:teuthology.orchestra.run.smithi114.stdout:PRETTY_NAME="CentOS Stream 9" 2024-04-24T12:56:02.174 INFO:teuthology.orchestra.run.smithi114.stdout:ANSI_COLOR="0;31" 2024-04-24T12:56:02.174 INFO:teuthology.orchestra.run.smithi114.stdout:LOGO="fedora-logo-icon" 2024-04-24T12:56:02.174 INFO:teuthology.orchestra.run.smithi114.stdout:CPE_NAME="cpe:/o:centos:centos:9" 2024-04-24T12:56:02.174 INFO:teuthology.orchestra.run.smithi114.stdout:HOME_URL="https://centos.org/" 2024-04-24T12:56:02.174 INFO:teuthology.orchestra.run.smithi114.stdout:BUG_REPORT_URL="https://bugzilla.redhat.com/" 2024-04-24T12:56:02.174 INFO:teuthology.orchestra.run.smithi114.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 9" 2024-04-24T12:56:02.174 INFO:teuthology.orchestra.run.smithi114.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream" 2024-04-24T12:56:02.175 INFO:teuthology.lock.ops:Updating smithi114.front.sepia.ceph.com on lock server 2024-04-24T12:56:02.193 INFO:teuthology.run_tasks:Running task internal.serialize_remote_roles... 2024-04-24T12:56:02.213 INFO:teuthology.run_tasks:Running task internal.check_conflict... 2024-04-24T12:56:02.238 INFO:teuthology.task.internal:Checking for old test directory... 2024-04-24T12:56:02.238 DEBUG:teuthology.orchestra.run.smithi069:> test '!' -e /home/ubuntu/cephtest 2024-04-24T12:56:02.243 DEBUG:teuthology.orchestra.run.smithi114:> test '!' -e /home/ubuntu/cephtest 2024-04-24T12:56:02.263 INFO:teuthology.run_tasks:Running task internal.check_ceph_data... 2024-04-24T12:56:02.274 INFO:teuthology.task.internal:Checking for non-empty /var/lib/ceph... 2024-04-24T12:56:02.274 DEBUG:teuthology.orchestra.run.smithi069:> test -z $(ls -A /var/lib/ceph) 2024-04-24T12:56:02.305 DEBUG:teuthology.orchestra.run.smithi114:> test -z $(ls -A /var/lib/ceph) 2024-04-24T12:56:02.429 INFO:teuthology.run_tasks:Running task internal.vm_setup... 2024-04-24T12:56:02.514 INFO:teuthology.run_tasks:Running task kernel... 2024-04-24T12:56:02.532 INFO:teuthology.task.kernel:normalize config orig: {'kdb': 1, 'sha1': 'distro'} 2024-04-24T12:56:02.532 INFO:teuthology.task.kernel:config {'host.a': {'kdb': 1, 'sha1': 'distro'}, 'host.b': {'kdb': 1, 'sha1': 'distro'}}, timeout 300 2024-04-24T12:56:02.533 DEBUG:teuthology.orchestra.run.smithi069:> test -f /run/.containerenv -o -f /.dockerenv 2024-04-24T12:56:02.533 DEBUG:teuthology.orchestra.run.smithi114:> test -f /run/.containerenv -o -f /.dockerenv 2024-04-24T12:56:02.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-24T12:56:02.551 DEBUG:teuthology.orchestra.run.smithi069:> uname -r 2024-04-24T12:56:02.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-24T12:56:02.552 DEBUG:teuthology.orchestra.run.smithi114:> uname -r 2024-04-24T12:56:02.608 INFO:teuthology.orchestra.run.smithi069.stdout:5.14.0-437.el9.x86_64 2024-04-24T12:56:02.609 INFO:teuthology.task.kernel:Running kernel on smithi069: 5.14.0-437.el9.x86_64 2024-04-24T12:56:02.609 DEBUG:teuthology.orchestra.run.smithi069:> sudo yum install -y kernel 2024-04-24T12:56:02.610 INFO:teuthology.orchestra.run.smithi114.stdout:5.14.0-437.el9.x86_64 2024-04-24T12:56:02.610 INFO:teuthology.task.kernel:Running kernel on smithi114: 5.14.0-437.el9.x86_64 2024-04-24T12:56:02.610 DEBUG:teuthology.orchestra.run.smithi114:> sudo yum install -y kernel 2024-04-24T12:56:04.924 INFO:teuthology.orchestra.run.smithi069.stdout:CentOS Stream 9 - BaseOS 12 MB/s | 8.0 MB 00:00 2024-04-24T12:56:05.969 INFO:teuthology.orchestra.run.smithi114.stdout:CentOS Stream 9 - BaseOS 4.8 MB/s | 8.0 MB 00:01 2024-04-24T12:56:09.920 INFO:teuthology.orchestra.run.smithi114.stdout:CentOS Stream 9 - AppStream 6.6 MB/s | 19 MB 00:02 2024-04-24T12:56:11.474 INFO:teuthology.orchestra.run.smithi069.stdout:CentOS Stream 9 - AppStream 3.5 MB/s | 19 MB 00:05 2024-04-24T12:56:13.928 INFO:teuthology.orchestra.run.smithi114.stdout:CentOS Stream 9 - CRB 14 MB/s | 6.2 MB 00:00 2024-04-24T12:56:15.674 INFO:teuthology.orchestra.run.smithi069.stdout:CentOS Stream 9 - CRB 11 MB/s | 6.2 MB 00:00 2024-04-24T12:56:16.204 INFO:teuthology.orchestra.run.smithi114.stdout:CentOS Stream 9 - Extras packages 15 kB/s | 16 kB 00:01 2024-04-24T12:56:17.447 INFO:teuthology.orchestra.run.smithi114.stdout:Extra Packages for Enterprise Linux 19 MB/s | 21 MB 00:01 2024-04-24T12:56:17.992 INFO:teuthology.orchestra.run.smithi069.stdout:CentOS Stream 9 - Extras packages 14 kB/s | 16 kB 00:01 2024-04-24T12:56:19.193 INFO:teuthology.orchestra.run.smithi069.stdout:Extra Packages for Enterprise Linux 19 MB/s | 21 MB 00:01 2024-04-24T12:56:22.456 INFO:teuthology.orchestra.run.smithi114.stdout:lab-extras 26 kB/s | 1.7 kB 00:00 2024-04-24T12:56:24.276 INFO:teuthology.orchestra.run.smithi069.stdout:lab-extras 13 kB/s | 1.7 kB 00:00 2024-04-24T12:56:24.328 INFO:teuthology.orchestra.run.smithi114.stdout:Package kernel-5.14.0-239.el9.x86_64 is already installed. 2024-04-24T12:56:24.329 INFO:teuthology.orchestra.run.smithi114.stdout:Package kernel-5.14.0-437.el9.x86_64 is already installed. 2024-04-24T12:56:24.422 INFO:teuthology.orchestra.run.smithi114.stdout:Dependencies resolved. 2024-04-24T12:56:24.423 INFO:teuthology.orchestra.run.smithi114.stdout:Nothing to do. 2024-04-24T12:56:24.423 INFO:teuthology.orchestra.run.smithi114.stdout:Complete! 2024-04-24T12:56:24.535 DEBUG:teuthology.orchestra.run.smithi114:> echo no | sudo yum reinstall kernel || true 2024-04-24T12:56:25.027 INFO:teuthology.orchestra.run.smithi114.stdout:Last metadata expiration check: 0:00:03 ago on Wed 24 Apr 2024 12:56:22 PM UTC. 2024-04-24T12:56:25.153 INFO:teuthology.orchestra.run.smithi114.stdout:Dependencies resolved. 2024-04-24T12:56:25.165 INFO:teuthology.orchestra.run.smithi114.stdout:================================================================================ 2024-04-24T12:56:25.165 INFO:teuthology.orchestra.run.smithi114.stdout: Package Architecture Version Repository Size 2024-04-24T12:56:25.165 INFO:teuthology.orchestra.run.smithi114.stdout:================================================================================ 2024-04-24T12:56:25.166 INFO:teuthology.orchestra.run.smithi114.stdout:Reinstalling: 2024-04-24T12:56:25.166 INFO:teuthology.orchestra.run.smithi114.stdout: kernel x86_64 5.14.0-437.el9 baseos 255 k 2024-04-24T12:56:25.166 INFO:teuthology.orchestra.run.smithi114.stdout: 2024-04-24T12:56:25.166 INFO:teuthology.orchestra.run.smithi114.stdout:Transaction Summary 2024-04-24T12:56:25.166 INFO:teuthology.orchestra.run.smithi114.stdout:================================================================================ 2024-04-24T12:56:25.166 INFO:teuthology.orchestra.run.smithi114.stdout: 2024-04-24T12:56:25.166 INFO:teuthology.orchestra.run.smithi114.stdout:Total download size: 255 k 2024-04-24T12:56:25.166 INFO:teuthology.orchestra.run.smithi114.stdout:Installed size: 0 2024-04-24T12:56:25.166 INFO:teuthology.orchestra.run.smithi114.stderr:Operation aborted. 2024-04-24T12:56:25.217 INFO:teuthology.orchestra.run.smithi114.stdout:Is this ok [y/N]: 2024-04-24T12:56:25.218 DEBUG:teuthology.orchestra.run.smithi114:> sudo yum reinstall -y kernel || true 2024-04-24T12:56:25.709 INFO:teuthology.orchestra.run.smithi114.stdout:Last metadata expiration check: 0:00:03 ago on Wed 24 Apr 2024 12:56:22 PM UTC. 2024-04-24T12:56:25.825 INFO:teuthology.orchestra.run.smithi114.stdout:Dependencies resolved. 2024-04-24T12:56:25.826 INFO:teuthology.orchestra.run.smithi114.stdout:================================================================================ 2024-04-24T12:56:25.826 INFO:teuthology.orchestra.run.smithi114.stdout: Package Architecture Version Repository Size 2024-04-24T12:56:25.826 INFO:teuthology.orchestra.run.smithi114.stdout:================================================================================ 2024-04-24T12:56:25.826 INFO:teuthology.orchestra.run.smithi114.stdout:Reinstalling: 2024-04-24T12:56:25.826 INFO:teuthology.orchestra.run.smithi114.stdout: kernel x86_64 5.14.0-437.el9 baseos 255 k 2024-04-24T12:56:25.826 INFO:teuthology.orchestra.run.smithi114.stdout: 2024-04-24T12:56:25.826 INFO:teuthology.orchestra.run.smithi114.stdout:Transaction Summary 2024-04-24T12:56:25.827 INFO:teuthology.orchestra.run.smithi114.stdout:================================================================================ 2024-04-24T12:56:25.827 INFO:teuthology.orchestra.run.smithi114.stdout: 2024-04-24T12:56:25.827 INFO:teuthology.orchestra.run.smithi114.stdout:Total download size: 255 k 2024-04-24T12:56:25.827 INFO:teuthology.orchestra.run.smithi114.stdout:Installed size: 0 2024-04-24T12:56:25.827 INFO:teuthology.orchestra.run.smithi114.stdout:Downloading Packages: 2024-04-24T12:56:26.140 INFO:teuthology.orchestra.run.smithi069.stdout:Package kernel-5.14.0-239.el9.x86_64 is already installed. 2024-04-24T12:56:26.141 INFO:teuthology.orchestra.run.smithi069.stdout:Package kernel-5.14.0-437.el9.x86_64 is already installed. 2024-04-24T12:56:26.183 INFO:teuthology.orchestra.run.smithi114.stdout:kernel-5.14.0-437.el9.x86_64.rpm 917 kB/s | 255 kB 00:00 2024-04-24T12:56:26.183 INFO:teuthology.orchestra.run.smithi114.stdout:-------------------------------------------------------------------------------- 2024-04-24T12:56:26.183 INFO:teuthology.orchestra.run.smithi114.stdout:Total 715 kB/s | 255 kB 00:00 2024-04-24T12:56:26.198 INFO:teuthology.orchestra.run.smithi114.stdout:Running transaction check 2024-04-24T12:56:26.227 INFO:teuthology.orchestra.run.smithi069.stdout:Dependencies resolved. 2024-04-24T12:56:26.228 INFO:teuthology.orchestra.run.smithi069.stdout:Nothing to do. 2024-04-24T12:56:26.229 INFO:teuthology.orchestra.run.smithi069.stdout:Complete! 2024-04-24T12:56:26.323 DEBUG:teuthology.orchestra.run.smithi069:> echo no | sudo yum reinstall kernel || true 2024-04-24T12:56:26.392 INFO:teuthology.orchestra.run.smithi114.stdout:Transaction check succeeded. 2024-04-24T12:56:26.392 INFO:teuthology.orchestra.run.smithi114.stdout:Running transaction test 2024-04-24T12:56:26.397 INFO:teuthology.orchestra.run.smithi114.stdout:Transaction test succeeded. 2024-04-24T12:56:26.398 INFO:teuthology.orchestra.run.smithi114.stdout:Running transaction 2024-04-24T12:56:26.528 INFO:teuthology.orchestra.run.smithi114.stdout: Preparing : 1/1 2024-04-24T12:56:26.609 INFO:teuthology.orchestra.run.smithi114.stdout: Reinstalling : kernel-5.14.0-437.el9.x86_64 1/2 2024-04-24T12:56:26.609 INFO:teuthology.orchestra.run.smithi114.stdout: Cleanup : kernel-5.14.0-437.el9.x86_64 2/2 2024-04-24T12:56:26.823 INFO:teuthology.orchestra.run.smithi069.stdout:Last metadata expiration check: 0:00:02 ago on Wed 24 Apr 2024 12:56:24 PM UTC. 2024-04-24T12:56:26.949 INFO:teuthology.orchestra.run.smithi069.stdout:Dependencies resolved. 2024-04-24T12:56:26.964 INFO:teuthology.orchestra.run.smithi069.stdout:================================================================================ 2024-04-24T12:56:26.964 INFO:teuthology.orchestra.run.smithi069.stdout: Package Architecture Version Repository Size 2024-04-24T12:56:26.964 INFO:teuthology.orchestra.run.smithi069.stdout:================================================================================ 2024-04-24T12:56:26.965 INFO:teuthology.orchestra.run.smithi069.stdout:Reinstalling: 2024-04-24T12:56:26.965 INFO:teuthology.orchestra.run.smithi069.stdout: kernel x86_64 5.14.0-437.el9 baseos 255 k 2024-04-24T12:56:26.965 INFO:teuthology.orchestra.run.smithi069.stdout: 2024-04-24T12:56:26.965 INFO:teuthology.orchestra.run.smithi069.stdout:Transaction Summary 2024-04-24T12:56:26.965 INFO:teuthology.orchestra.run.smithi069.stdout:================================================================================ 2024-04-24T12:56:26.965 INFO:teuthology.orchestra.run.smithi069.stdout: 2024-04-24T12:56:26.966 INFO:teuthology.orchestra.run.smithi069.stdout:Total download size: 255 k 2024-04-24T12:56:26.966 INFO:teuthology.orchestra.run.smithi069.stdout:Installed size: 0 2024-04-24T12:56:26.967 INFO:teuthology.orchestra.run.smithi069.stderr:Operation aborted. 2024-04-24T12:56:27.019 INFO:teuthology.orchestra.run.smithi069.stdout:Is this ok [y/N]: 2024-04-24T12:56:27.019 DEBUG:teuthology.orchestra.run.smithi069:> sudo yum reinstall -y kernel || true 2024-04-24T12:56:27.334 INFO:teuthology.orchestra.run.smithi114.stdout: Running scriptlet: kernel-5.14.0-437.el9.x86_64 2/2 2024-04-24T12:56:27.335 INFO:teuthology.orchestra.run.smithi114.stdout: Verifying : kernel-5.14.0-437.el9.x86_64 1/2 2024-04-24T12:56:27.512 INFO:teuthology.orchestra.run.smithi069.stdout:Last metadata expiration check: 0:00:03 ago on Wed 24 Apr 2024 12:56:24 PM UTC. 2024-04-24T12:56:27.627 INFO:teuthology.orchestra.run.smithi069.stdout:Dependencies resolved. 2024-04-24T12:56:27.628 INFO:teuthology.orchestra.run.smithi069.stdout:================================================================================ 2024-04-24T12:56:27.628 INFO:teuthology.orchestra.run.smithi069.stdout: Package Architecture Version Repository Size 2024-04-24T12:56:27.628 INFO:teuthology.orchestra.run.smithi069.stdout:================================================================================ 2024-04-24T12:56:27.628 INFO:teuthology.orchestra.run.smithi069.stdout:Reinstalling: 2024-04-24T12:56:27.628 INFO:teuthology.orchestra.run.smithi069.stdout: kernel x86_64 5.14.0-437.el9 baseos 255 k 2024-04-24T12:56:27.628 INFO:teuthology.orchestra.run.smithi069.stdout: 2024-04-24T12:56:27.629 INFO:teuthology.orchestra.run.smithi069.stdout:Transaction Summary 2024-04-24T12:56:27.629 INFO:teuthology.orchestra.run.smithi069.stdout:================================================================================ 2024-04-24T12:56:27.629 INFO:teuthology.orchestra.run.smithi069.stdout: 2024-04-24T12:56:27.629 INFO:teuthology.orchestra.run.smithi069.stdout:Total download size: 255 k 2024-04-24T12:56:27.629 INFO:teuthology.orchestra.run.smithi069.stdout:Installed size: 0 2024-04-24T12:56:27.629 INFO:teuthology.orchestra.run.smithi069.stdout:Downloading Packages: 2024-04-24T12:56:28.025 INFO:teuthology.orchestra.run.smithi114.stdout: Verifying : kernel-5.14.0-437.el9.x86_64 2/2 2024-04-24T12:56:28.025 INFO:teuthology.orchestra.run.smithi114.stdout: 2024-04-24T12:56:28.025 INFO:teuthology.orchestra.run.smithi114.stdout:Reinstalled: 2024-04-24T12:56:28.025 INFO:teuthology.orchestra.run.smithi114.stdout: kernel-5.14.0-437.el9.x86_64 2024-04-24T12:56:28.025 INFO:teuthology.orchestra.run.smithi114.stdout: 2024-04-24T12:56:28.026 INFO:teuthology.orchestra.run.smithi114.stdout:Complete! 2024-04-24T12:56:28.205 DEBUG:teuthology.orchestra.run.smithi114:> rpm -q kernel | sort -rV | head -n 1 2024-04-24T12:56:28.293 INFO:teuthology.orchestra.run.smithi114.stdout:kernel-5.14.0-437.el9.x86_64 2024-04-24T12:56:28.294 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-437.el9.x86_64 2024-04-24T12:56:28.294 INFO:teuthology.task.kernel:Newest distro kernel installed and running 2024-04-24T12:56:28.294 INFO:teuthology.task.kernel:Enabling kdb on host.b... 2024-04-24T12:56:28.294 DEBUG:teuthology.orchestra.run.smithi114:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2024-04-24T12:56:28.331 INFO:teuthology.orchestra.run.smithi114.stdout:ttyS1 2024-04-24T12:56:28.348 DEBUG:teuthology.parallel:result is None 2024-04-24T12:56:28.378 INFO:teuthology.orchestra.run.smithi069.stdout:kernel-5.14.0-437.el9.x86_64.rpm 535 kB/s | 255 kB 00:00 2024-04-24T12:56:28.378 INFO:teuthology.orchestra.run.smithi069.stdout:-------------------------------------------------------------------------------- 2024-04-24T12:56:28.378 INFO:teuthology.orchestra.run.smithi069.stdout:Total 340 kB/s | 255 kB 00:00 2024-04-24T12:56:28.399 INFO:teuthology.orchestra.run.smithi069.stdout:Running transaction check 2024-04-24T12:56:28.589 INFO:teuthology.orchestra.run.smithi069.stdout:Transaction check succeeded. 2024-04-24T12:56:28.589 INFO:teuthology.orchestra.run.smithi069.stdout:Running transaction test 2024-04-24T12:56:28.594 INFO:teuthology.orchestra.run.smithi069.stdout:Transaction test succeeded. 2024-04-24T12:56:28.594 INFO:teuthology.orchestra.run.smithi069.stdout:Running transaction 2024-04-24T12:56:28.733 INFO:teuthology.orchestra.run.smithi069.stdout: Preparing : 1/1 2024-04-24T12:56:28.821 INFO:teuthology.orchestra.run.smithi069.stdout: Reinstalling : kernel-5.14.0-437.el9.x86_64 1/2 2024-04-24T12:56:28.821 INFO:teuthology.orchestra.run.smithi069.stdout: Cleanup : kernel-5.14.0-437.el9.x86_64 2/2 2024-04-24T12:56:29.563 INFO:teuthology.orchestra.run.smithi069.stdout: Running scriptlet: kernel-5.14.0-437.el9.x86_64 2/2 2024-04-24T12:56:29.563 INFO:teuthology.orchestra.run.smithi069.stdout: Verifying : kernel-5.14.0-437.el9.x86_64 1/2 2024-04-24T12:56:30.269 INFO:teuthology.orchestra.run.smithi069.stdout: Verifying : kernel-5.14.0-437.el9.x86_64 2/2 2024-04-24T12:56:30.269 INFO:teuthology.orchestra.run.smithi069.stdout: 2024-04-24T12:56:30.269 INFO:teuthology.orchestra.run.smithi069.stdout:Reinstalled: 2024-04-24T12:56:30.270 INFO:teuthology.orchestra.run.smithi069.stdout: kernel-5.14.0-437.el9.x86_64 2024-04-24T12:56:30.270 INFO:teuthology.orchestra.run.smithi069.stdout: 2024-04-24T12:56:30.270 INFO:teuthology.orchestra.run.smithi069.stdout:Complete! 2024-04-24T12:56:30.434 DEBUG:teuthology.orchestra.run.smithi069:> rpm -q kernel | sort -rV | head -n 1 2024-04-24T12:56:30.481 INFO:teuthology.orchestra.run.smithi069.stdout:kernel-5.14.0-437.el9.x86_64 2024-04-24T12:56:30.481 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-437.el9.x86_64 2024-04-24T12:56:30.481 INFO:teuthology.task.kernel:Newest distro kernel installed and running 2024-04-24T12:56:30.481 INFO:teuthology.task.kernel:Enabling kdb on host.a... 2024-04-24T12:56:30.482 DEBUG:teuthology.orchestra.run.smithi069:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2024-04-24T12:56:30.520 INFO:teuthology.orchestra.run.smithi069.stdout:ttyS1 2024-04-24T12:56:30.538 DEBUG:teuthology.parallel:result is None 2024-04-24T12:56:30.538 INFO:teuthology.run_tasks:Running task internal.base... 2024-04-24T12:56:30.546 INFO:teuthology.task.internal:Creating test directory... 2024-04-24T12:56:30.546 DEBUG:teuthology.orchestra.run.smithi069:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2024-04-24T12:56:30.580 DEBUG:teuthology.orchestra.run.smithi114:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2024-04-24T12:56:30.598 INFO:teuthology.run_tasks:Running task internal.archive_upload... 2024-04-24T12:56:30.605 INFO:teuthology.run_tasks:Running task internal.archive... 2024-04-24T12:56:30.612 INFO:teuthology.task.internal:Creating archive directory... 2024-04-24T12:56:30.612 DEBUG:teuthology.orchestra.run.smithi069:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2024-04-24T12:56:30.636 DEBUG:teuthology.orchestra.run.smithi114:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2024-04-24T12:56:30.670 INFO:teuthology.run_tasks:Running task internal.coredump... 2024-04-24T12:56:30.677 INFO:teuthology.task.internal:Enabling coredump saving... 2024-04-24T12:56:30.678 DEBUG:teuthology.orchestra.run.smithi069:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2024-04-24T12:56:30.701 DEBUG:teuthology.orchestra.run.smithi114:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2024-04-24T12:56:30.725 INFO:teuthology.orchestra.run.smithi069.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-04-24T12:56:30.734 INFO:teuthology.orchestra.run.smithi069.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-04-24T12:56:30.742 INFO:teuthology.orchestra.run.smithi114.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-04-24T12:56:30.752 INFO:teuthology.orchestra.run.smithi114.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-04-24T12:56:30.754 INFO:teuthology.run_tasks:Running task internal.sudo... 2024-04-24T12:56:30.788 INFO:teuthology.task.internal:Configuring sudo... 2024-04-24T12:56:30.788 DEBUG:teuthology.orchestra.run.smithi069:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2024-04-24T12:56:30.791 DEBUG:teuthology.orchestra.run.smithi114:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2024-04-24T12:56:30.821 INFO:teuthology.run_tasks:Running task internal.syslog... 2024-04-24T12:56:30.831 INFO:teuthology.task.internal.syslog:Starting syslog monitoring... 2024-04-24T12:56:30.831 DEBUG:teuthology.orchestra.run.smithi069:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2024-04-24T12:56:30.855 DEBUG:teuthology.orchestra.run.smithi114:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2024-04-24T12:56:30.878 DEBUG:teuthology.orchestra.run.smithi069:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2024-04-24T12:56:30.949 DEBUG:teuthology.orchestra.run.smithi069:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2024-04-24T12:56:30.997 DEBUG:teuthology.orchestra.run.smithi069:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2024-04-24T12:56:31.081 DEBUG:teuthology.orchestra.run.smithi069:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2024-04-24T12:56:31.117 DEBUG:teuthology.orchestra.run.smithi069:> set -ex 2024-04-24T12:56:31.117 DEBUG:teuthology.orchestra.run.smithi069:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2024-04-24T12:56:31.181 DEBUG:teuthology.orchestra.run.smithi114:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2024-04-24T12:56:31.233 DEBUG:teuthology.orchestra.run.smithi114:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2024-04-24T12:56:31.690 DEBUG:teuthology.orchestra.run.smithi114:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2024-04-24T12:56:31.774 DEBUG:teuthology.orchestra.run.smithi114:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2024-04-24T12:56:31.817 DEBUG:teuthology.orchestra.run.smithi114:> set -ex 2024-04-24T12:56:31.817 DEBUG:teuthology.orchestra.run.smithi114:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2024-04-24T12:56:31.884 DEBUG:teuthology.orchestra.run.smithi069:> sudo service rsyslog restart 2024-04-24T12:56:31.886 DEBUG:teuthology.orchestra.run.smithi114:> sudo service rsyslog restart 2024-04-24T12:56:31.922 INFO:teuthology.orchestra.run.smithi069.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-04-24T12:56:31.957 INFO:teuthology.orchestra.run.smithi114.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-04-24T12:56:32.480 INFO:teuthology.run_tasks:Running task internal.timer... 2024-04-24T12:56:32.489 INFO:teuthology.task.internal:Starting timer... 2024-04-24T12:56:32.489 INFO:teuthology.run_tasks:Running task pcp... 2024-04-24T12:56:32.499 INFO:teuthology.run_tasks:Running task selinux... 2024-04-24T12:56:32.508 DEBUG:teuthology.task:Applying overrides for task selinux: {'allowlist': ['scontext=system_u:system_r:logrotate_t:s0', 'scontext=system_u:system_r:getty_t:s0']} 2024-04-24T12:56:32.508 DEBUG:teuthology.orchestra.run.smithi069:> sudo service auditd rotate 2024-04-24T12:56:32.559 INFO:teuthology.orchestra.run.smithi069.stdout:Rotating logs: 2024-04-24T12:56:32.560 DEBUG:teuthology.orchestra.run.smithi114:> sudo service auditd rotate 2024-04-24T12:56:32.623 INFO:teuthology.orchestra.run.smithi114.stdout:Rotating logs: 2024-04-24T12:56:32.625 DEBUG:teuthology.task.selinux:Getting current SELinux state 2024-04-24T12:56:32.625 DEBUG:teuthology.orchestra.run.smithi069:> /usr/sbin/getenforce 2024-04-24T12:56:32.650 INFO:teuthology.orchestra.run.smithi069.stdout:Permissive 2024-04-24T12:56:32.650 DEBUG:teuthology.orchestra.run.smithi114:> /usr/sbin/getenforce 2024-04-24T12:56:32.671 INFO:teuthology.orchestra.run.smithi114.stdout:Permissive 2024-04-24T12:56:32.671 DEBUG:teuthology.task.selinux:Existing SELinux modes: {'ubuntu@smithi069.front.sepia.ceph.com': 'permissive', 'ubuntu@smithi114.front.sepia.ceph.com': 'permissive'} 2024-04-24T12:56:32.671 DEBUG:teuthology.orchestra.run.smithi069:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 -e scontext=system_u:system_r:getty_t:s0 2024-04-24T12:56:32.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-24T12:56:32.716 DEBUG:teuthology.orchestra.run.smithi114:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 -e scontext=system_u:system_r:getty_t:s0 2024-04-24T12:56:32.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-24T12:56:32.741 INFO:teuthology.task.selinux:Putting SELinux into permissive mode 2024-04-24T12:56:32.741 DEBUG:teuthology.orchestra.run.smithi069:> sudo /usr/sbin/setenforce permissive 2024-04-24T12:56:32.780 DEBUG:teuthology.orchestra.run.smithi114:> sudo /usr/sbin/setenforce permissive 2024-04-24T12:56:32.805 INFO:teuthology.run_tasks:Running task ansible.cephlab... 2024-04-24T12:56:32.816 DEBUG:teuthology.repo_utils:Setting repo remote to https://git.ceph.com/ceph-cm-ansible.git 2024-04-24T12:56:32.822 INFO:teuthology.repo_utils:Fetching git.ceph.com_ceph-cm-ansible_main from origin 2024-04-24T12:56:32.952 DEBUG:teuthology.repo_utils:Resetting repo at /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main to origin/main 2024-04-24T12:56:32.994 INFO:teuthology.task.ansible:Playbook: [{'import_playbook': 'ansible_managed.yml'}, {'import_playbook': 'teuthology.yml'}, {'hosts': 'testnodes', 'tasks': [{'set_fact': {'ran_from_cephlab_playbook': True}}]}, {'import_playbook': 'testnodes.yml'}, {'import_playbook': 'container-host.yml'}, {'import_playbook': 'cobbler.yml'}, {'import_playbook': 'paddles.yml'}, {'import_playbook': 'pulpito.yml'}, {'hosts': 'testnodes', 'become': True, 'tasks': [{'name': 'Touch /ceph-qa-ready', 'file': {'path': '/ceph-qa-ready', 'state': 'touch'}, 'when': 'ran_from_cephlab_playbook|bool'}]}] 2024-04-24T12:56:32.996 DEBUG:teuthology.task.ansible:Running ansible-playbook -v --extra-vars '{"ansible_ssh_user": "ubuntu"}' -i /etc/ansible/hosts --limit smithi069.front.sepia.ceph.com,smithi114.front.sepia.ceph.com /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main/cephlab.yml 2024-04-24T13:00:02.776 DEBUG:teuthology.task.ansible:Reconnecting to [Remote(name='ubuntu@smithi069.front.sepia.ceph.com'), Remote(name='ubuntu@smithi114.front.sepia.ceph.com')] 2024-04-24T13:00:02.777 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi069.front.sepia.ceph.com' 2024-04-24T13:00:02.779 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi069.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-24T13:00:02.909 DEBUG:teuthology.orchestra.run.smithi069:> true 2024-04-24T13:00:02.991 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi069.front.sepia.ceph.com' 2024-04-24T13:00:02.992 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi114.front.sepia.ceph.com' 2024-04-24T13:00:02.992 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi114.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-24T13:00:03.077 DEBUG:teuthology.orchestra.run.smithi114:> true 2024-04-24T13:00:03.171 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi114.front.sepia.ceph.com' 2024-04-24T13:00:03.171 INFO:teuthology.run_tasks:Running task clock... 2024-04-24T13:00:03.449 INFO:teuthology.task.clock:Syncing clocks and checking initial clock skew... 2024-04-24T13:00:03.449 INFO:teuthology.orchestra.run:Running command with timeout 360 2024-04-24T13:00:03.449 DEBUG:teuthology.orchestra.run.smithi069:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-04-24T13:00:03.453 INFO:teuthology.orchestra.run:Running command with timeout 360 2024-04-24T13:00:03.453 DEBUG:teuthology.orchestra.run.smithi114:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-04-24T13:00:03.480 INFO:teuthology.orchestra.run.smithi069.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2024-04-24T13:00:03.484 INFO:teuthology.orchestra.run.smithi114.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2024-04-24T13:00:03.495 INFO:teuthology.orchestra.run.smithi069.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2024-04-24T13:00:03.499 INFO:teuthology.orchestra.run.smithi114.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2024-04-24T13:00:03.518 INFO:teuthology.orchestra.run.smithi069.stderr:sudo: ntpd: command not found 2024-04-24T13:00:03.523 INFO:teuthology.orchestra.run.smithi114.stderr:sudo: ntpd: command not found 2024-04-24T13:00:03.529 INFO:teuthology.orchestra.run.smithi069.stdout:506 Cannot talk to daemon 2024-04-24T13:00:03.534 INFO:teuthology.orchestra.run.smithi114.stdout:506 Cannot talk to daemon 2024-04-24T13:00:03.542 INFO:teuthology.orchestra.run.smithi069.stderr:Failed to start ntp.service: Unit ntp.service not found. 2024-04-24T13:00:03.548 INFO:teuthology.orchestra.run.smithi114.stderr:Failed to start ntp.service: Unit ntp.service not found. 2024-04-24T13:00:03.555 INFO:teuthology.orchestra.run.smithi069.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2024-04-24T13:00:03.562 INFO:teuthology.orchestra.run.smithi114.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2024-04-24T13:00:03.614 INFO:teuthology.orchestra.run.smithi114.stderr:bash: line 1: ntpq: command not found 2024-04-24T13:00:03.615 INFO:teuthology.orchestra.run.smithi069.stderr:bash: line 1: ntpq: command not found 2024-04-24T13:00:03.617 INFO:teuthology.orchestra.run.smithi114.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-04-24T13:00:03.617 INFO:teuthology.orchestra.run.smithi114.stdout:=============================================================================== 2024-04-24T13:00:03.617 INFO:teuthology.orchestra.run.smithi114.stdout:^? hv01.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-04-24T13:00:03.618 INFO:teuthology.orchestra.run.smithi114.stdout:^? hv02.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-04-24T13:00:03.618 INFO:teuthology.orchestra.run.smithi114.stdout:^? hv03.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-04-24T13:00:03.618 INFO:teuthology.orchestra.run.smithi114.stdout:^? hv04.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-04-24T13:00:03.619 INFO:teuthology.orchestra.run.smithi069.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-04-24T13:00:03.619 INFO:teuthology.orchestra.run.smithi069.stdout:=============================================================================== 2024-04-24T13:00:03.619 INFO:teuthology.orchestra.run.smithi069.stdout:^? hv01.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-04-24T13:00:03.619 INFO:teuthology.orchestra.run.smithi069.stdout:^? hv02.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-04-24T13:00:03.619 INFO:teuthology.orchestra.run.smithi069.stdout:^? hv03.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-04-24T13:00:03.619 INFO:teuthology.orchestra.run.smithi069.stdout:^? hv04.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-04-24T13:00:03.620 INFO:teuthology.run_tasks:Running task cephadm... 2024-04-24T13:00:04.107 INFO:tasks.cephadm:Config: {'cephadm_branch': 'v17.2.0', 'cephadm_git_url': 'https://github.com/ceph/ceph', 'image': 'quay.io/ceph/ceph:v17.2.0', 'roleless': True, 'conf': {'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}, 'osd': {'debug ms': 1, 'debug osd': 20, 'osd shutdown pgref assert': True}}, 'flavor': 'default', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)'], 'log-only-match': ['CEPHADM_'], 'sha1': '23fcfb96e7e1a49d12a94e3f87a8e3f06db2a1ec'} 2024-04-24T13:00:04.107 INFO:tasks.cephadm:Cluster image is quay.io/ceph/ceph:v17.2.0 2024-04-24T13:00:04.108 INFO:tasks.cephadm:Cluster fsid is 9150213e-023a-11ef-bc93-c7b262605968 2024-04-24T13:00:04.108 INFO:tasks.cephadm:Choosing monitor IPs and ports... 2024-04-24T13:00:04.108 INFO:tasks.cephadm:No mon roles; fabricating mons 2024-04-24T13:00:04.108 INFO:tasks.cephadm:Monitor IPs: {'mon.smithi069': '172.21.15.69', 'mon.smithi114': '172.21.15.114'} 2024-04-24T13:00:04.108 INFO:tasks.cephadm:Normalizing hostnames... 2024-04-24T13:00:04.109 DEBUG:teuthology.orchestra.run.smithi069:> sudo hostname $(hostname -s) 2024-04-24T13:00:04.134 DEBUG:teuthology.orchestra.run.smithi114:> sudo hostname $(hostname -s) 2024-04-24T13:00:04.162 INFO:tasks.cephadm:Downloading cephadm (repo https://github.com/ceph/ceph ref v17.2.0)... 2024-04-24T13:00:04.163 DEBUG:teuthology.orchestra.run.smithi069:> curl --silent https://raw.githubusercontent.com/ceph/ceph/v17.2.0/src/cephadm/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2024-04-24T13:00:04.458 INFO:teuthology.orchestra.run.smithi069.stdout:-rw-r--r--. 1 ubuntu ubuntu 320521 Apr 24 13:00 /home/ubuntu/cephtest/cephadm 2024-04-24T13:00:04.459 DEBUG:teuthology.orchestra.run.smithi114:> curl --silent https://raw.githubusercontent.com/ceph/ceph/v17.2.0/src/cephadm/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2024-04-24T13:00:04.764 INFO:teuthology.orchestra.run.smithi114.stdout:-rw-r--r--. 1 ubuntu ubuntu 320521 Apr 24 13:00 /home/ubuntu/cephtest/cephadm 2024-04-24T13:00:04.765 DEBUG:teuthology.orchestra.run.smithi069:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2024-04-24T13:00:04.782 DEBUG:teuthology.orchestra.run.smithi114:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2024-04-24T13:00:04.810 INFO:tasks.cephadm:Pulling image quay.io/ceph/ceph:v17.2.0 on all hosts... 2024-04-24T13:00:04.810 DEBUG:teuthology.orchestra.run.smithi069:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 pull 2024-04-24T13:00:04.825 DEBUG:teuthology.orchestra.run.smithi114:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 pull 2024-04-24T13:00:06.329 INFO:teuthology.orchestra.run.smithi114.stderr:Pulling container image quay.io/ceph/ceph:v17.2.0... 2024-04-24T13:00:06.349 INFO:teuthology.orchestra.run.smithi069.stderr:Pulling container image quay.io/ceph/ceph:v17.2.0... 2024-04-24T13:00:24.910 INFO:teuthology.orchestra.run.smithi114.stdout:{ 2024-04-24T13:00:24.910 INFO:teuthology.orchestra.run.smithi114.stdout: "ceph_version": "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)", 2024-04-24T13:00:24.910 INFO:teuthology.orchestra.run.smithi114.stdout: "image_id": "e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9", 2024-04-24T13:00:24.911 INFO:teuthology.orchestra.run.smithi114.stdout: "repo_digests": [ 2024-04-24T13:00:24.911 INFO:teuthology.orchestra.run.smithi114.stdout: "quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a", 2024-04-24T13:00:24.911 INFO:teuthology.orchestra.run.smithi114.stdout: "quay.io/ceph/ceph@sha256:cb4d698cb769b6aba05bf6ef04f41a7fe694160140347576e13bd9348514b667" 2024-04-24T13:00:24.911 INFO:teuthology.orchestra.run.smithi114.stdout: ] 2024-04-24T13:00:24.911 INFO:teuthology.orchestra.run.smithi114.stdout:} 2024-04-24T13:00:26.855 INFO:teuthology.orchestra.run.smithi069.stdout:{ 2024-04-24T13:00:26.855 INFO:teuthology.orchestra.run.smithi069.stdout: "ceph_version": "ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable)", 2024-04-24T13:00:26.855 INFO:teuthology.orchestra.run.smithi069.stdout: "image_id": "e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9", 2024-04-24T13:00:26.855 INFO:teuthology.orchestra.run.smithi069.stdout: "repo_digests": [ 2024-04-24T13:00:26.855 INFO:teuthology.orchestra.run.smithi069.stdout: "quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a", 2024-04-24T13:00:26.855 INFO:teuthology.orchestra.run.smithi069.stdout: "quay.io/ceph/ceph@sha256:cb4d698cb769b6aba05bf6ef04f41a7fe694160140347576e13bd9348514b667" 2024-04-24T13:00:26.855 INFO:teuthology.orchestra.run.smithi069.stdout: ] 2024-04-24T13:00:26.855 INFO:teuthology.orchestra.run.smithi069.stdout:} 2024-04-24T13:00:26.873 DEBUG:teuthology.orchestra.run.smithi069:> sudo mkdir -p /etc/ceph 2024-04-24T13:00:26.925 DEBUG:teuthology.orchestra.run.smithi114:> sudo mkdir -p /etc/ceph 2024-04-24T13:00:26.959 DEBUG:teuthology.orchestra.run.smithi069:> sudo chmod 777 /etc/ceph 2024-04-24T13:00:26.982 DEBUG:teuthology.orchestra.run.smithi114:> sudo chmod 777 /etc/ceph 2024-04-24T13:00:27.026 INFO:tasks.cephadm:Writing seed config... 2024-04-24T13:00:27.027 INFO:tasks.cephadm: override: [mgr] debug mgr = 20 2024-04-24T13:00:27.027 INFO:tasks.cephadm: override: [mgr] debug ms = 1 2024-04-24T13:00:27.027 INFO:tasks.cephadm: override: [mon] debug mon = 20 2024-04-24T13:00:27.027 INFO:tasks.cephadm: override: [mon] debug ms = 1 2024-04-24T13:00:27.027 INFO:tasks.cephadm: override: [mon] debug paxos = 20 2024-04-24T13:00:27.027 INFO:tasks.cephadm: override: [osd] debug ms = 1 2024-04-24T13:00:27.027 INFO:tasks.cephadm: override: [osd] debug osd = 20 2024-04-24T13:00:27.027 INFO:tasks.cephadm: override: [osd] osd shutdown pgref assert = True 2024-04-24T13:00:27.028 DEBUG:teuthology.orchestra.run.smithi069:> set -ex 2024-04-24T13:00:27.028 DEBUG:teuthology.orchestra.run.smithi069:> dd of=/home/ubuntu/cephtest/seed.ceph.conf 2024-04-24T13:00:27.042 DEBUG:tasks.cephadm:Final config: [global] # make logging friendly to teuthology log_to_file = true log_to_stderr = false log to journald = false mon cluster log to file = true mon cluster log file level = debug mon clock drift allowed = 1.000 # replicate across OSDs, not hosts osd crush chooseleaf type = 0 #osd pool default size = 2 osd pool default erasure code profile = plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd # enable some debugging auth debug = true ms die on old message = true ms die on bug = true debug asserts on shutdown = true # adjust warnings mon max pg per osd = 10000# >= luminous mon pg warn max object skew = 0 mon osd allow primary affinity = true mon osd allow pg remap = true mon warn on legacy crush tunables = false mon warn on crush straw calc version zero = false mon warn on no sortbitwise = false mon warn on osd down out interval zero = false mon warn on too few osds = false mon_warn_on_pool_pg_num_not_power_of_two = false # disable pg_autoscaler by default for new pools osd_pool_default_pg_autoscale_mode = off # tests delete pools mon allow pool delete = true fsid = 9150213e-023a-11ef-bc93-c7b262605968 [osd] osd scrub load threshold = 5.0 osd scrub max interval = 600 osd mclock profile = high_recovery_ops osd recover clone overlap = true osd recovery max chunk = 1048576 osd deep scrub update digest min age = 30 osd map max advance = 10 osd memory target autotune = true # debugging osd debug shutdown = true osd debug op order = true osd debug verify stray on activate = true osd debug pg log writeout = true osd debug verify cached snaps = true osd debug verify missing on start = true osd debug misdirected ops = true osd op queue = debug_random osd op queue cut off = debug_random osd shutdown pgref assert = True bdev debug aio = true osd sloppy crc = true debug ms = 1 debug osd = 20 [mgr] mon reweight min pgs per osd = 4 mon reweight min bytes per osd = 10 mgr/telemetry/nag = false debug mgr = 20 debug ms = 1 [mon] mon data avail warn = 5 mon mgr mkfs grace = 240 mon reweight min pgs per osd = 4 mon osd reporter subtree level = osd mon osd prime pg temp = true mon reweight min bytes per osd = 10 # rotate auth tickets quickly to exercise renewal paths auth mon ticket ttl = 660# 11m auth service ticket ttl = 240# 4m # don't complain about global id reclaim mon_warn_on_insecure_global_id_reclaim = false mon_warn_on_insecure_global_id_reclaim_allowed = false debug mon = 20 debug ms = 1 debug paxos = 20 [client.rgw] rgw cache enabled = true rgw enable ops log = true rgw enable usage log = true 2024-04-24T13:00:27.043 DEBUG:teuthology.orchestra.run.smithi069:mon.smithi069> sudo journalctl -f -n 0 -u ceph-9150213e-023a-11ef-bc93-c7b262605968@mon.smithi069.service 2024-04-24T13:00:27.085 INFO:tasks.cephadm:Bootstrapping... 2024-04-24T13:00:27.085 DEBUG:teuthology.orchestra.run.smithi069:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 -v bootstrap --fsid 9150213e-023a-11ef-bc93-c7b262605968 --config /home/ubuntu/cephtest/seed.ceph.conf --output-config /etc/ceph/ceph.conf --output-keyring /etc/ceph/ceph.client.admin.keyring --output-pub-ssh-key /home/ubuntu/cephtest/ceph.pub --mon-ip 172.21.15.69 --skip-admin-label && sudo chmod +r /etc/ceph/ceph.client.admin.keyring 2024-04-24T13:00:27.278 INFO:teuthology.orchestra.run.smithi069.stderr:-------------------------------------------------------------------------------- 2024-04-24T13:00:27.279 INFO:teuthology.orchestra.run.smithi069.stderr:cephadm ['--image', 'quay.io/ceph/ceph:v17.2.0', '-v', 'bootstrap', '--fsid', '9150213e-023a-11ef-bc93-c7b262605968', '--config', '/home/ubuntu/cephtest/seed.ceph.conf', '--output-config', '/etc/ceph/ceph.conf', '--output-keyring', '/etc/ceph/ceph.client.admin.keyring', '--output-pub-ssh-key', '/home/ubuntu/cephtest/ceph.pub', '--mon-ip', '172.21.15.69', '--skip-admin-label'] 2024-04-24T13:00:27.301 INFO:teuthology.orchestra.run.smithi069.stderr:/bin/podman: 4.9.4-dev 2024-04-24T13:00:27.304 INFO:teuthology.orchestra.run.smithi069.stderr:Verifying podman|docker is present... 2024-04-24T13:00:27.326 INFO:teuthology.orchestra.run.smithi069.stderr:/bin/podman: 4.9.4-dev 2024-04-24T13:00:27.327 INFO:teuthology.orchestra.run.smithi069.stderr:Verifying lvm2 is present... 2024-04-24T13:00:27.327 INFO:teuthology.orchestra.run.smithi069.stderr:Verifying time synchronization is in place... 2024-04-24T13:00:27.335 INFO:teuthology.orchestra.run.smithi069.stderr:systemctl: Failed to get unit file state for chrony.service: No such file or directory 2024-04-24T13:00:27.341 INFO:teuthology.orchestra.run.smithi069.stderr:systemctl: inactive 2024-04-24T13:00:27.347 INFO:teuthology.orchestra.run.smithi069.stderr:systemctl: enabled 2024-04-24T13:00:27.353 INFO:teuthology.orchestra.run.smithi069.stderr:systemctl: active 2024-04-24T13:00:27.353 INFO:teuthology.orchestra.run.smithi069.stderr:Unit chronyd.service is enabled and running 2024-04-24T13:00:27.353 INFO:teuthology.orchestra.run.smithi069.stderr:Repeating the final host check... 2024-04-24T13:00:27.374 INFO:teuthology.orchestra.run.smithi069.stderr:/bin/podman: 4.9.4-dev 2024-04-24T13:00:27.376 INFO:teuthology.orchestra.run.smithi069.stderr:podman (/bin/podman) version 4.9.4 is present 2024-04-24T13:00:27.376 INFO:teuthology.orchestra.run.smithi069.stderr:systemctl is present 2024-04-24T13:00:27.376 INFO:teuthology.orchestra.run.smithi069.stderr:lvcreate is present 2024-04-24T13:00:27.382 INFO:teuthology.orchestra.run.smithi069.stderr:systemctl: Failed to get unit file state for chrony.service: No such file or directory 2024-04-24T13:00:27.388 INFO:teuthology.orchestra.run.smithi069.stderr:systemctl: inactive 2024-04-24T13:00:27.394 INFO:teuthology.orchestra.run.smithi069.stderr:systemctl: enabled 2024-04-24T13:00:27.400 INFO:teuthology.orchestra.run.smithi069.stderr:systemctl: active 2024-04-24T13:00:27.401 INFO:teuthology.orchestra.run.smithi069.stderr:Unit chronyd.service is enabled and running 2024-04-24T13:00:27.401 INFO:teuthology.orchestra.run.smithi069.stderr:Host looks OK 2024-04-24T13:00:27.401 INFO:teuthology.orchestra.run.smithi069.stderr:Cluster fsid: 9150213e-023a-11ef-bc93-c7b262605968 2024-04-24T13:00:27.401 INFO:teuthology.orchestra.run.smithi069.stderr:Acquiring lock 140286633341664 on /run/cephadm/9150213e-023a-11ef-bc93-c7b262605968.lock 2024-04-24T13:00:27.401 INFO:teuthology.orchestra.run.smithi069.stderr:Lock 140286633341664 acquired on /run/cephadm/9150213e-023a-11ef-bc93-c7b262605968.lock 2024-04-24T13:00:27.402 INFO:teuthology.orchestra.run.smithi069.stderr:Verifying IP 172.21.15.69 port 3300 ... 2024-04-24T13:00:27.402 INFO:teuthology.orchestra.run.smithi069.stderr:Verifying IP 172.21.15.69 port 6789 ... 2024-04-24T13:00:27.402 INFO:teuthology.orchestra.run.smithi069.stderr:Base mon IP is 172.21.15.69, final addrv is [v2:172.21.15.69:3300,v1:172.21.15.69:6789] 2024-04-24T13:00:27.405 INFO:teuthology.orchestra.run.smithi069.stderr:/sbin/ip: default via 172.21.15.254 dev enp3s0f1 proto dhcp src 172.21.15.69 metric 100 2024-04-24T13:00:27.405 INFO:teuthology.orchestra.run.smithi069.stderr:/sbin/ip: 172.21.0.0/20 dev enp3s0f1 proto kernel scope link src 172.21.15.69 metric 100 2024-04-24T13:00:27.407 INFO:teuthology.orchestra.run.smithi069.stderr:/sbin/ip: ::1 dev lo proto kernel metric 256 pref medium 2024-04-24T13:00:27.407 INFO:teuthology.orchestra.run.smithi069.stderr:/sbin/ip: fe80::/64 dev enp3s0f1 proto kernel metric 1024 pref medium 2024-04-24T13:00:27.407 INFO:teuthology.orchestra.run.smithi069.stderr:/sbin/ip: default via fe80::327c:5e00:6487:71e0 dev enp3s0f1 proto ra metric 100 pref medium 2024-04-24T13:00:27.410 INFO:teuthology.orchestra.run.smithi069.stderr:/sbin/ip: 1: lo: mtu 65536 state UNKNOWN qlen 1000 2024-04-24T13:00:27.410 INFO:teuthology.orchestra.run.smithi069.stderr:/sbin/ip: inet6 ::1/128 scope host 2024-04-24T13:00:27.410 INFO:teuthology.orchestra.run.smithi069.stderr:/sbin/ip: valid_lft forever preferred_lft forever 2024-04-24T13:00:27.410 INFO:teuthology.orchestra.run.smithi069.stderr:/sbin/ip: 5: enp3s0f1: mtu 1500 state UP qlen 1000 2024-04-24T13:00:27.410 INFO:teuthology.orchestra.run.smithi069.stderr:/sbin/ip: inet6 fe80::c126:1983:9bc9:c998/64 scope link noprefixroute 2024-04-24T13:00:27.410 INFO:teuthology.orchestra.run.smithi069.stderr:/sbin/ip: valid_lft forever preferred_lft forever 2024-04-24T13:00:27.411 INFO:teuthology.orchestra.run.smithi069.stderr:Mon IP `172.21.15.69` is in CIDR network `172.21.0.0/20` 2024-04-24T13:00:27.411 INFO:teuthology.orchestra.run.smithi069.stderr:- internal network (--cluster-network) has not been provided, OSD replication will default to the public_network 2024-04-24T13:00:27.411 INFO:teuthology.orchestra.run.smithi069.stderr:Pulling container image quay.io/ceph/ceph:v17.2.0... 2024-04-24T13:00:27.515 INFO:teuthology.orchestra.run.smithi069.stderr:/bin/podman: Trying to pull quay.io/ceph/ceph:v17.2.0... 2024-04-24T13:00:28.096 INFO:teuthology.orchestra.run.smithi069.stderr:/bin/podman: Getting image source signatures 2024-04-24T13:00:28.096 INFO:teuthology.orchestra.run.smithi069.stderr:/bin/podman: Copying blob sha256:c32ab78b488d0c72f64eded765c0cf6b5bf2c75dab66cb62a9d367fa6ec42513 2024-04-24T13:00:28.096 INFO:teuthology.orchestra.run.smithi069.stderr:/bin/podman: Copying blob sha256:599d07cb321ff0a3c82224e1138fc685793fa69b93ed5780415751a5f7e4b8c2 2024-04-24T13:00:28.096 INFO:teuthology.orchestra.run.smithi069.stderr:/bin/podman: Copying blob sha256:a70843738bb77e1ab9c1f85969ebdfa55f178e746be081d1cb4f94011f69eb7c 2024-04-24T13:00:28.096 INFO:teuthology.orchestra.run.smithi069.stderr:/bin/podman: Copying blob sha256:33ca8fff7868c4dc0c11e09bca97c720eb9cfbab7221216754367dd8de70388a 2024-04-24T13:00:28.096 INFO:teuthology.orchestra.run.smithi069.stderr:/bin/podman: Copying blob sha256:89b4a75bc2d8500f15463747507c9623df43886c134463e7f0527e70900e7a7b 2024-04-24T13:00:28.096 INFO:teuthology.orchestra.run.smithi069.stderr:/bin/podman: Copying config sha256:e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 2024-04-24T13:00:30.057 INFO:teuthology.orchestra.run.smithi069.stderr:/bin/podman: Writing manifest to image destination 2024-04-24T13:00:31.444 INFO:teuthology.orchestra.run.smithi069.stderr:/bin/podman: e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 2024-04-24T13:00:35.339 INFO:teuthology.orchestra.run.smithi069.stderr:ceph: ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable) 2024-04-24T13:00:37.436 INFO:teuthology.orchestra.run.smithi069.stderr:Ceph version: ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable) 2024-04-24T13:00:37.436 INFO:teuthology.orchestra.run.smithi069.stderr:Extracting ceph user uid/gid from container image... 2024-04-24T13:00:37.852 INFO:teuthology.orchestra.run.smithi069.stderr:stat: 167 167 2024-04-24T13:00:38.457 INFO:teuthology.orchestra.run.smithi069.stderr:Creating initial keys... 2024-04-24T13:00:39.115 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph-authtool: AQB3AilmD16bARAAFblYQKy3Z4ylY6BKi69ZNg== 2024-04-24T13:00:40.046 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph-authtool: AQB3AilmWUN3ORAA31y20dqxiIpZ3StMkP1Ycw== 2024-04-24T13:00:40.951 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph-authtool: AQB4AilmgbyzMxAA2s9MGjIpcd/S/IO/x8yhBA== 2024-04-24T13:00:41.541 INFO:teuthology.orchestra.run.smithi069.stderr:Creating initial monmap... 2024-04-24T13:00:41.948 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/monmaptool: /usr/bin/monmaptool: monmap file /tmp/monmap 2024-04-24T13:00:41.949 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/monmaptool: setting min_mon_release = octopus 2024-04-24T13:00:41.949 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/monmaptool: /usr/bin/monmaptool: set fsid to 9150213e-023a-11ef-bc93-c7b262605968 2024-04-24T13:00:41.949 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/monmaptool: /usr/bin/monmaptool: writing epoch 0 to /tmp/monmap (1 monitors) 2024-04-24T13:00:43.837 INFO:teuthology.orchestra.run.smithi069.stderr:monmaptool for smithi069 [v2:172.21.15.69:3300,v1:172.21.15.69:6789] on /usr/bin/monmaptool: monmap file /tmp/monmap 2024-04-24T13:00:43.837 INFO:teuthology.orchestra.run.smithi069.stderr:setting min_mon_release = octopus 2024-04-24T13:00:43.837 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/monmaptool: set fsid to 9150213e-023a-11ef-bc93-c7b262605968 2024-04-24T13:00:43.837 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/monmaptool: writing epoch 0 to /tmp/monmap (1 monitors) 2024-04-24T13:00:43.837 INFO:teuthology.orchestra.run.smithi069.stderr: 2024-04-24T13:00:43.838 INFO:teuthology.orchestra.run.smithi069.stderr:Creating mon... 2024-04-24T13:00:44.833 INFO:teuthology.orchestra.run.smithi069.stderr:create mon.smithi069 on 2024-04-24T13:00:45.039 INFO:teuthology.orchestra.run.smithi069.stderr:systemctl: Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target → /etc/systemd/system/ceph.target. 2024-04-24T13:00:45.181 INFO:teuthology.orchestra.run.smithi069.stderr:systemctl: Created symlink /etc/systemd/system/multi-user.target.wants/ceph-9150213e-023a-11ef-bc93-c7b262605968.target → /etc/systemd/system/ceph-9150213e-023a-11ef-bc93-c7b262605968.target. 2024-04-24T13:00:45.181 INFO:teuthology.orchestra.run.smithi069.stderr:systemctl: Created symlink /etc/systemd/system/ceph.target.wants/ceph-9150213e-023a-11ef-bc93-c7b262605968.target → /etc/systemd/system/ceph-9150213e-023a-11ef-bc93-c7b262605968.target. 2024-04-24T13:00:45.481 INFO:teuthology.orchestra.run.smithi069.stderr:systemctl: Failed to reset failed state of unit ceph-9150213e-023a-11ef-bc93-c7b262605968@mon.smithi069.service: Unit ceph-9150213e-023a-11ef-bc93-c7b262605968@mon.smithi069.service not loaded. 2024-04-24T13:00:45.489 INFO:teuthology.orchestra.run.smithi069.stderr:systemctl: Created symlink /etc/systemd/system/ceph-9150213e-023a-11ef-bc93-c7b262605968.target.wants/ceph-9150213e-023a-11ef-bc93-c7b262605968@mon.smithi069.service → /etc/systemd/system/ceph-9150213e-023a-11ef-bc93-c7b262605968@.service. 2024-04-24T13:00:46.138 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:00:46 smithi069 podman[21436]: 2024-04-24 13:00:46.099330674 +0000 UTC m=+0.332445742 container start e5ee01443dd5c939d5500e875ae1792f2382fe6e27401aa718228de948b25a0a (image=quay.io/ceph/ceph:v17.2.0, name=ceph-9150213e-023a-11ef-bc93-c7b262605968-mon-smithi069, io.openshift.tags=base centos centos-stream, CEPH_POINT_RELEASE=-17.2.0, release=754, maintainer=Guillaume Abrioux , io.k8s.display-name=CentOS Stream 8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.buildah.version=1.19.8, com.redhat.component=centos-stream-container, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, ceph=True, GIT_BRANCH=HEAD, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, distribution-scope=public, vendor=Red Hat, Inc., GIT_CLEAN=True, architecture=x86_64, name=centos-stream, RELEASE=HEAD, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., version=8, build-date=2022-05-03T08:36:31.336870, io.openshift.expose-services=, GIT_REPO=https://github.com/ceph/ceph-container.git, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, vcs-type=git) 2024-04-24T13:00:46.138 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:00:46 smithi069 ceph-mon[21454]: mkfs 9150213e-023a-11ef-bc93-c7b262605968 2024-04-24T13:00:46.207 INFO:teuthology.orchestra.run.smithi069.stderr:systemctl: disabled 2024-04-24T13:00:46.214 INFO:teuthology.orchestra.run.smithi069.stderr:systemctl: inactive 2024-04-24T13:00:46.214 INFO:teuthology.orchestra.run.smithi069.stderr:firewalld.service is not enabled 2024-04-24T13:00:46.214 INFO:teuthology.orchestra.run.smithi069.stderr:Not possible to enable service . firewalld.service is not available 2024-04-24T13:00:46.215 INFO:teuthology.orchestra.run.smithi069.stderr:Waiting for mon to start... 2024-04-24T13:00:46.215 INFO:teuthology.orchestra.run.smithi069.stderr:Waiting for mon... 2024-04-24T13:00:46.488 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:00:46 smithi069 ceph-mon[21454]: mon.smithi069 is new leader, mons smithi069 in quorum (ranks 0) 2024-04-24T13:00:46.489 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:00:46 smithi069 bash[21436]: e5ee01443dd5c939d5500e875ae1792f2382fe6e27401aa718228de948b25a0a 2024-04-24T13:00:46.489 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:00:46 smithi069 systemd[1]: Started Ceph mon.smithi069 for 9150213e-023a-11ef-bc93-c7b262605968. 2024-04-24T13:00:46.760 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: cluster: 2024-04-24T13:00:46.761 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: id: 9150213e-023a-11ef-bc93-c7b262605968 2024-04-24T13:00:46.761 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: health: HEALTH_OK 2024-04-24T13:00:46.761 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: 2024-04-24T13:00:46.761 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: services: 2024-04-24T13:00:46.761 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: mon: 1 daemons, quorum smithi069 (age 0.624891s) 2024-04-24T13:00:46.761 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: mgr: no daemons active 2024-04-24T13:00:46.761 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: osd: 0 osds: 0 up, 0 in 2024-04-24T13:00:46.761 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: 2024-04-24T13:00:46.761 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: data: 2024-04-24T13:00:46.762 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: pools: 0 pools, 0 pgs 2024-04-24T13:00:46.762 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: objects: 0 objects, 0 B 2024-04-24T13:00:46.762 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: usage: 0 B used, 0 B / 0 B avail 2024-04-24T13:00:46.762 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: pgs: 2024-04-24T13:00:46.762 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: 2024-04-24T13:00:47.342 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:00:47 smithi069 ceph-mon[21454]: mon.smithi069 is new leader, mons smithi069 in quorum (ranks 0) 2024-04-24T13:00:47.342 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:00:47 smithi069 ceph-mon[21454]: monmap e1: 1 mons at {smithi069=[v2:172.21.15.69:3300/0,v1:172.21.15.69:6789/0]} 2024-04-24T13:00:47.342 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:00:47 smithi069 ceph-mon[21454]: fsmap 2024-04-24T13:00:47.342 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:00:47 smithi069 ceph-mon[21454]: osdmap e1: 0 total, 0 up, 0 in 2024-04-24T13:00:47.342 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:00:47 smithi069 ceph-mon[21454]: mgrmap e1: no daemons active 2024-04-24T13:00:47.342 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:00:47 smithi069 ceph-mon[21454]: from='client.? 172.21.15.69:0/1416524171' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-04-24T13:00:47.344 INFO:teuthology.orchestra.run.smithi069.stderr:mon is available 2024-04-24T13:00:47.344 INFO:teuthology.orchestra.run.smithi069.stderr:Assimilating anything we can from ceph.conf... 2024-04-24T13:00:47.871 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: 2024-04-24T13:00:47.872 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: [global] 2024-04-24T13:00:47.872 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: fsid = 9150213e-023a-11ef-bc93-c7b262605968 2024-04-24T13:00:47.872 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: mon_host = [v2:172.21.15.69:3300,v1:172.21.15.69:6789] 2024-04-24T13:00:47.872 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: mon_osd_allow_pg_remap = true 2024-04-24T13:00:47.872 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: mon_osd_allow_primary_affinity = true 2024-04-24T13:00:47.872 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: mon_warn_on_no_sortbitwise = false 2024-04-24T13:00:47.872 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: osd_crush_chooseleaf_type = 0 2024-04-24T13:00:47.872 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: 2024-04-24T13:00:47.872 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: [mgr] 2024-04-24T13:00:47.872 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: mgr/telemetry/nag = false 2024-04-24T13:00:47.872 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: 2024-04-24T13:00:47.872 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: [osd] 2024-04-24T13:00:47.872 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: osd_map_max_advance = 10 2024-04-24T13:00:47.873 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: osd_sloppy_crc = true 2024-04-24T13:00:48.359 INFO:teuthology.orchestra.run.smithi069.stderr:Generating new minimal ceph.conf... 2024-04-24T13:00:48.480 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:00:48 smithi069 ceph-mon[21454]: from='client.? 172.21.15.69:0/998536468' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2024-04-24T13:00:48.481 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:00:48 smithi069 ceph-mon[21454]: from='client.? 172.21.15.69:0/998536468' entity='client.admin' cmd='[{"prefix": "config assimilate-conf"}]': finished 2024-04-24T13:00:49.438 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:00:49 smithi069 ceph-mon[21454]: from='client.? 172.21.15.69:0/3873340499' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-24T13:00:49.683 INFO:teuthology.orchestra.run.smithi069.stderr:Restarting the monitor... 2024-04-24T13:00:50.100 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:00:49 smithi069 systemd[1]: Stopping Ceph mon.smithi069 for 9150213e-023a-11ef-bc93-c7b262605968... 2024-04-24T13:00:50.101 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:00:49 smithi069 bash[21674]: Error: no container with name or ID "ceph-9150213e-023a-11ef-bc93-c7b262605968-mon.smithi069" found: no such container 2024-04-24T13:00:50.101 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:00:49 smithi069 ceph-9150213e-023a-11ef-bc93-c7b262605968-mon-smithi069[21450]: 2024-04-24T13:00:49.851+0000 7f9cc8f20700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.smithi069 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-04-24T13:00:50.101 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:00:49 smithi069 ceph-9150213e-023a-11ef-bc93-c7b262605968-mon-smithi069[21450]: 2024-04-24T13:00:49.851+0000 7f9cc8f20700 -1 mon.smithi069@0(leader) e1 *** Got Signal Terminated *** 2024-04-24T13:00:50.377 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:00:50 smithi069 podman[21687]: 2024-04-24 13:00:50.107672374 +0000 UTC m=+0.361838028 container died e5ee01443dd5c939d5500e875ae1792f2382fe6e27401aa718228de948b25a0a (image=quay.io/ceph/ceph:v17.2.0, name=ceph-9150213e-023a-11ef-bc93-c7b262605968-mon-smithi069, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.k8s.display-name=CentOS Stream 8, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_REPO=https://github.com/ceph/ceph-container.git, RELEASE=HEAD, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, version=8, io.buildah.version=1.19.8, vcs-type=git, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, io.openshift.tags=base centos centos-stream, io.openshift.expose-services=, maintainer=Guillaume Abrioux , com.redhat.component=centos-stream-container, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, CEPH_POINT_RELEASE=-17.2.0, distribution-scope=public, vendor=Red Hat, Inc., name=centos-stream, architecture=x86_64, ceph=True, GIT_BRANCH=HEAD, release=754, GIT_CLEAN=True, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, build-date=2022-05-03T08:36:31.336870, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac) 2024-04-24T13:00:50.797 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:00:50 smithi069 podman[21687]: 2024-04-24 13:00:50.379099442 +0000 UTC m=+0.633265102 container cleanup e5ee01443dd5c939d5500e875ae1792f2382fe6e27401aa718228de948b25a0a (image=quay.io/ceph/ceph:v17.2.0, name=ceph-9150213e-023a-11ef-bc93-c7b262605968-mon-smithi069, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., RELEASE=HEAD, GIT_BRANCH=HEAD, distribution-scope=public, build-date=2022-05-03T08:36:31.336870, io.openshift.expose-services=, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vendor=Red Hat, Inc., com.redhat.license_terms=https://centos.org/legal/licensing-policy/, CEPH_POINT_RELEASE=-17.2.0, com.redhat.component=centos-stream-container, io.buildah.version=1.19.8, io.k8s.display-name=CentOS Stream 8, architecture=x86_64, release=754, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, name=centos-stream, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_REPO=https://github.com/ceph/ceph-container.git, ceph=True, GIT_CLEAN=True, io.openshift.tags=base centos centos-stream, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, maintainer=Guillaume Abrioux , url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vcs-type=git, version=8) 2024-04-24T13:00:50.798 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:00:50 smithi069 bash[21687]: ceph-9150213e-023a-11ef-bc93-c7b262605968-mon-smithi069 2024-04-24T13:00:50.798 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:00:50 smithi069 bash[21716]: Error: no container with name or ID "ceph-9150213e-023a-11ef-bc93-c7b262605968-mon.smithi069" found: no such container 2024-04-24T13:00:50.798 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:00:50 smithi069 podman[21701]: 2024-04-24 13:00:50.560427955 +0000 UTC m=+0.454564938 container remove e5ee01443dd5c939d5500e875ae1792f2382fe6e27401aa718228de948b25a0a (image=quay.io/ceph/ceph:v17.2.0, name=ceph-9150213e-023a-11ef-bc93-c7b262605968-mon-smithi069, ceph=True, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, distribution-scope=public, GIT_BRANCH=HEAD, io.buildah.version=1.19.8, build-date=2022-05-03T08:36:31.336870, version=8, com.redhat.component=centos-stream-container, release=754, io.openshift.tags=base centos centos-stream, vcs-type=git, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., CEPH_POINT_RELEASE=-17.2.0, RELEASE=HEAD, maintainer=Guillaume Abrioux , architecture=x86_64, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.k8s.display-name=CentOS Stream 8, name=centos-stream, vendor=Red Hat, Inc., summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_CLEAN=True, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, GIT_REPO=https://github.com/ceph/ceph-container.git, io.openshift.expose-services=) 2024-04-24T13:00:50.798 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:00:50 smithi069 systemd[1]: ceph-9150213e-023a-11ef-bc93-c7b262605968@mon.smithi069.service: Deactivated successfully. 2024-04-24T13:00:50.798 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:00:50 smithi069 systemd[1]: Stopped Ceph mon.smithi069 for 9150213e-023a-11ef-bc93-c7b262605968. 2024-04-24T13:00:50.798 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:00:50 smithi069 systemd[1]: Starting Ceph mon.smithi069 for 9150213e-023a-11ef-bc93-c7b262605968... 2024-04-24T13:00:51.094 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:00:50 smithi069 podman[21781]: 2024-04-24 13:00:50.703164848 +0000 UTC m=+0.018707857 image pull e1d6a67b021eb077ee22bf650f1a9fb1980a2cf5c36bdb9cba9eac6de8f702d9 quay.io/ceph/ceph:v17.2.0 2024-04-24T13:00:51.094 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:00:50 smithi069 podman[21781]: 2024-04-24 13:00:50.80257861 +0000 UTC m=+0.118121615 container create 82aada682e0492920095b94da619855b8ceb3ca28c06247cbdccba51b632f64b (image=quay.io/ceph/ceph:v17.2.0, name=ceph-9150213e-023a-11ef-bc93-c7b262605968-mon-smithi069, GIT_BRANCH=HEAD, ceph=True, CEPH_POINT_RELEASE=-17.2.0, RELEASE=HEAD, io.k8s.display-name=CentOS Stream 8, io.openshift.expose-services=, GIT_REPO=https://github.com/ceph/ceph-container.git, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-type=git, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, io.buildah.version=1.19.8, version=8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, distribution-scope=public, maintainer=Guillaume Abrioux , GIT_CLEAN=True, architecture=x86_64, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, vendor=Red Hat, Inc., name=centos-stream, com.redhat.component=centos-stream-container, build-date=2022-05-03T08:36:31.336870, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, release=754, io.openshift.tags=base centos centos-stream) 2024-04-24T13:00:51.094 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:00:51 smithi069 podman[21781]: 2024-04-24 13:00:51.013138253 +0000 UTC m=+0.328681254 container init 82aada682e0492920095b94da619855b8ceb3ca28c06247cbdccba51b632f64b (image=quay.io/ceph/ceph:v17.2.0, name=ceph-9150213e-023a-11ef-bc93-c7b262605968-mon-smithi069, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, ceph=True, release=754, CEPH_POINT_RELEASE=-17.2.0, maintainer=Guillaume Abrioux , version=8, io.k8s.display-name=CentOS Stream 8, GIT_BRANCH=HEAD, RELEASE=HEAD, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, GIT_CLEAN=True, com.redhat.component=centos-stream-container, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.buildah.version=1.19.8, build-date=2022-05-03T08:36:31.336870, io.openshift.expose-services=, vcs-type=git, architecture=x86_64, vendor=Red Hat, Inc., io.openshift.tags=base centos centos-stream, GIT_REPO=https://github.com/ceph/ceph-container.git, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.license_terms=https://centos.org/legal/licensing-policy/, distribution-scope=public, name=centos-stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream) 2024-04-24T13:00:51.095 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:00:51 smithi069 podman[21781]: 2024-04-24 13:00:51.015933345 +0000 UTC m=+0.331476351 container start 82aada682e0492920095b94da619855b8ceb3ca28c06247cbdccba51b632f64b (image=quay.io/ceph/ceph:v17.2.0, name=ceph-9150213e-023a-11ef-bc93-c7b262605968-mon-smithi069, com.redhat.component=centos-stream-container, build-date=2022-05-03T08:36:31.336870, GIT_BRANCH=HEAD, vendor=Red Hat, Inc., com.redhat.license_terms=https://centos.org/legal/licensing-policy/, io.k8s.display-name=CentOS Stream 8, io.openshift.expose-services=, RELEASE=HEAD, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, GIT_REPO=https://github.com/ceph/ceph-container.git, GIT_CLEAN=True, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., architecture=x86_64, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, distribution-scope=public, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=-17.2.0, name=centos-stream, io.buildah.version=1.19.8, version=8, vcs-type=git, release=754, io.openshift.tags=base centos centos-stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, ceph=True, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754) 2024-04-24T13:00:51.095 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:00:51 smithi069 ceph-mon[21800]: set uid:gid to 167:167 (ceph:ceph) 2024-04-24T13:00:51.095 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:00:51 smithi069 ceph-mon[21800]: ceph version 17.2.0 (43e2e60a7559d3f46c9d53f1ca875fd499a1e35e) quincy (stable), process ceph-mon, pid 2 2024-04-24T13:00:51.095 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:00:51 smithi069 ceph-mon[21800]: pidfile_write: ignore empty --pid-file 2024-04-24T13:00:51.095 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:00:51 smithi069 ceph-mon[21800]: load: jerasure load: lrc 2024-04-24T13:00:51.095 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:00:51 smithi069 ceph-mon[21800]: rocksdb: RocksDB version: 6.15.5 2024-04-24T13:00:51.095 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:00:51 smithi069 ceph-mon[21800]: rocksdb: Git sha rocksdb_build_git_sha:@0@ 2024-04-24T13:00:51.095 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:00:51 smithi069 ceph-mon[21800]: rocksdb: Compile date Apr 18 2022 2024-04-24T13:00:51.095 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:00:51 smithi069 ceph-mon[21800]: rocksdb: DB SUMMARY 2024-04-24T13:00:51.095 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:00:51 smithi069 ceph-mon[21800]: rocksdb: DB Session ID: 2XO3DVAU93V0AP7OENFO 2024-04-24T13:00:51.095 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:00:51 smithi069 ceph-mon[21800]: rocksdb: CURRENT file: CURRENT 2024-04-24T13:00:51.096 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:00:51 smithi069 ceph-mon[21800]: rocksdb: IDENTITY file: IDENTITY 2024-04-24T13:00:51.096 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:00:51 smithi069 ceph-mon[21800]: rocksdb: MANIFEST file: MANIFEST-000009 size: 131 Bytes 2024-04-24T13:00:51.096 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:00:51 smithi069 ceph-mon[21800]: rocksdb: SST files in /var/lib/ceph/mon/ceph-smithi069/store.db dir, Total Num: 1, files: 000008.sst 2024-04-24T13:00:51.096 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:00:51 smithi069 ceph-mon[21800]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-smithi069/store.db: 000010.log size: 82705 ; 2024-04-24T13:00:51.096 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:00:51 smithi069 ceph-mon[21800]: rocksdb: Options.error_if_exists: 0 2024-04-24T13:00:51.096 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:00:51 smithi069 ceph-mon[21800]: rocksdb: Options.create_if_missing: 0 2024-04-24T13:00:51.096 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:00:51 smithi069 ceph-mon[21800]: rocksdb: Options.paranoid_checks: 1 2024-04-24T13:00:51.097 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:00:51 smithi069 ceph-mon[21800]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-04-24T13:00:51.097 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:00:51 smithi069 ceph-mon[21800]: rocksdb: Options.env: 0x55c9facc6860 2024-04-24T13:00:51.097 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:00:51 smithi069 ceph-mon[21800]: rocksdb: Options.fs: Posix File System 2024-04-24T13:00:51.097 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:00:51 smithi069 ceph-mon[21800]: rocksdb: Options.info_log: 0x55c9fbe2bfa0 2024-04-24T13:00:51.097 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:00:51 smithi069 ceph-mon[21800]: rocksdb: Options.max_file_opening_threads: 16 2024-04-24T13:00:51.097 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:00:51 smithi069 ceph-mon[21800]: rocksdb: Options.statistics: (nil) 2024-04-24T13:00:51.097 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:00:51 smithi069 ceph-mon[21800]: rocksdb: Options.use_fsync: 0 2024-04-24T13:00:51.097 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:00:51 smithi069 ceph-mon[21800]: rocksdb: Options.max_log_file_size: 0 2024-04-24T13:00:51.097 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:00:51 smithi069 ceph-mon[21800]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-04-24T13:00:51.099 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:00:51 smithi069 ceph-mon[21800]: rocksdb: Options.log_file_time_to_roll: 0 2024-04-24T13:00:51.099 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:00:51 smithi069 ceph-mon[21800]: rocksdb: Options.keep_log_file_num: 1000 2024-04-24T13:00:51.099 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:00:51 smithi069 ceph-mon[21800]: rocksdb: Options.recycle_log_file_num: 0 2024-04-24T13:00:51.099 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:00:51 smithi069 ceph-mon[21800]: rocksdb: Options.allow_fallocate: 1 2024-04-24T13:00:51.099 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:00:51 smithi069 ceph-mon[21800]: rocksdb: Options.allow_mmap_reads: 0 2024-04-24T13:00:51.099 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:00:51 smithi069 ceph-mon[21800]: rocksdb: Options.allow_mmap_writes: 0 2024-04-24T13:00:51.099 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:00:51 smithi069 ceph-mon[21800]: rocksdb: Options.use_direct_reads: 0 2024-04-24T13:00:51.099 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:00:51 smithi069 ceph-mon[21800]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-04-24T13:00:51.099 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:00:51 smithi069 ceph-mon[21800]: rocksdb: Options.create_missing_column_families: 0 2024-04-24T13:00:51.100 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:00:51 smithi069 ceph-mon[21800]: rocksdb: Options.db_log_dir: 2024-04-24T13:00:51.100 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:00:51 smithi069 ceph-mon[21800]: rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-smithi069/store.db 2024-04-24T13:00:51.100 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:00:51 smithi069 ceph-mon[21800]: rocksdb: Options.table_cache_numshardbits: 6 2024-04-24T13:00:51.100 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:00:51 smithi069 ceph-mon[21800]: rocksdb: Options.WAL_ttl_seconds: 0 2024-04-24T13:00:51.100 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:00:51 smithi069 ceph-mon[21800]: rocksdb: Options.WAL_size_limit_MB: 0 2024-04-24T13:00:51.100 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:00:51 smithi069 ceph-mon[21800]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-04-24T13:00:51.100 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:00:51 smithi069 ceph-mon[21800]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-04-24T13:00:51.101 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:00:51 smithi069 ceph-mon[21800]: rocksdb: Options.is_fd_close_on_exec: 1 2024-04-24T13:00:51.101 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:00:51 smithi069 ceph-mon[21800]: rocksdb: Options.advise_random_on_open: 1 2024-04-24T13:00:51.101 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:00:51 smithi069 ceph-mon[21800]: rocksdb: Options.db_write_buffer_size: 0 2024-04-24T13:00:51.101 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:00:51 smithi069 ceph-mon[21800]: rocksdb: Options.write_buffer_manager: 0x55c9fbf1c330 2024-04-24T13:00:51.101 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:00:51 smithi069 ceph-mon[21800]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-04-24T13:00:51.101 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:00:51 smithi069 ceph-mon[21800]: rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2024-04-24T13:00:51.101 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:00:51 smithi069 ceph-mon[21800]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-04-24T13:00:51.101 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:00:51 smithi069 ceph-mon[21800]: rocksdb: Options.use_adaptive_mutex: 0 2024-04-24T13:00:51.101 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:00:51 smithi069 ceph-mon[21800]: rocksdb: Options.rate_limiter: (nil) 2024-04-24T13:00:51.102 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:00:51 smithi069 ceph-mon[21800]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-04-24T13:00:51.102 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:00:51 smithi069 ceph-mon[21800]: rocksdb: Options.wal_recovery_mode: 2 2024-04-24T13:00:51.102 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:00:51 smithi069 ceph-mon[21800]: rocksdb: Options.enable_thread_tracking: 0 2024-04-24T13:00:51.102 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:00:51 smithi069 ceph-mon[21800]: rocksdb: Options.enable_pipelined_write: 0 2024-04-24T13:00:51.102 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:00:51 smithi069 ceph-mon[21800]: rocksdb: Options.unordered_write: 0 2024-04-24T13:00:51.102 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:00:51 smithi069 ceph-mon[21800]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-04-24T13:00:51.102 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:00:51 smithi069 ceph-mon[21800]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-04-24T13:00:51.102 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:00:51 smithi069 ceph-mon[21800]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-04-24T13:00:51.102 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:00:51 smithi069 ceph-mon[21800]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-04-24T13:00:51.102 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:00:51 smithi069 ceph-mon[21800]: rocksdb: Options.row_cache: None 2024-04-24T13:00:51.103 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:00:51 smithi069 ceph-mon[21800]: rocksdb: Options.wal_filter: None 2024-04-24T13:00:51.103 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:00:51 smithi069 ceph-mon[21800]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-04-24T13:00:51.103 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:00:51 smithi069 ceph-mon[21800]: rocksdb: Options.allow_ingest_behind: 0 2024-04-24T13:00:51.103 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:00:51 smithi069 ceph-mon[21800]: rocksdb: Options.preserve_deletes: 0 2024-04-24T13:00:51.103 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:00:51 smithi069 ceph-mon[21800]: rocksdb: Options.two_write_queues: 0 2024-04-24T13:00:51.103 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:00:51 smithi069 ceph-mon[21800]: rocksdb: Options.manual_wal_flush: 0 2024-04-24T13:00:51.103 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:00:51 smithi069 ceph-mon[21800]: rocksdb: Options.atomic_flush: 0 2024-04-24T13:00:51.103 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:00:51 smithi069 ceph-mon[21800]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-04-24T13:00:51.103 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:00:51 smithi069 ceph-mon[21800]: rocksdb: Options.persist_stats_to_disk: 0 2024-04-24T13:00:51.103 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:00:51 smithi069 ceph-mon[21800]: rocksdb: Options.write_dbid_to_manifest: 0 2024-04-24T13:00:51.103 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:00:51 smithi069 ceph-mon[21800]: rocksdb: Options.log_readahead_size: 0 2024-04-24T13:00:51.103 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:00:51 smithi069 ceph-mon[21800]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-04-24T13:00:51.103 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:00:51 smithi069 ceph-mon[21800]: rocksdb: Options.best_efforts_recovery: 0 2024-04-24T13:00:51.103 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:00:51 smithi069 ceph-mon[21800]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-04-24T13:00:51.103 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:00:51 smithi069 ceph-mon[21800]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-04-24T13:00:51.104 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:00:51 smithi069 ceph-mon[21800]: rocksdb: Options.allow_data_in_errors: 0 2024-04-24T13:00:51.104 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:00:51 smithi069 ceph-mon[21800]: rocksdb: Options.db_host_id: __hostname__ 2024-04-24T13:00:51.104 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:00:51 smithi069 ceph-mon[21800]: rocksdb: Options.max_background_jobs: 2 2024-04-24T13:00:51.104 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:00:51 smithi069 ceph-mon[21800]: rocksdb: Options.max_background_compactions: -1 2024-04-24T13:00:51.104 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:00:51 smithi069 ceph-mon[21800]: rocksdb: Options.max_subcompactions: 1 2024-04-24T13:00:51.104 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:00:51 smithi069 ceph-mon[21800]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-04-24T13:00:51.104 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:00:51 smithi069 ceph-mon[21800]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-04-24T13:00:51.104 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:00:51 smithi069 ceph-mon[21800]: rocksdb: Options.delayed_write_rate : 16777216 2024-04-24T13:00:51.104 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:00:51 smithi069 ceph-mon[21800]: rocksdb: Options.max_total_wal_size: 0 2024-04-24T13:00:51.104 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:00:51 smithi069 ceph-mon[21800]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-04-24T13:00:51.104 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:00:51 smithi069 ceph-mon[21800]: rocksdb: Options.stats_dump_period_sec: 600 2024-04-24T13:00:51.104 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:00:51 smithi069 ceph-mon[21800]: rocksdb: Options.stats_persist_period_sec: 600 2024-04-24T13:00:51.104 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:00:51 smithi069 ceph-mon[21800]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-04-24T13:00:51.105 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:00:51 smithi069 ceph-mon[21800]: rocksdb: Options.max_open_files: -1 2024-04-24T13:00:51.105 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:00:51 smithi069 ceph-mon[21800]: rocksdb: Options.bytes_per_sync: 0 2024-04-24T13:00:51.105 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:00:51 smithi069 ceph-mon[21800]: rocksdb: Options.wal_bytes_per_sync: 0 2024-04-24T13:00:51.105 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:00:51 smithi069 ceph-mon[21800]: rocksdb: Options.strict_bytes_per_sync: 0 2024-04-24T13:00:51.105 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:00:51 smithi069 ceph-mon[21800]: rocksdb: Options.compaction_readahead_size: 0 2024-04-24T13:00:51.105 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:00:51 smithi069 ceph-mon[21800]: rocksdb: Options.max_background_flushes: -1 2024-04-24T13:00:51.105 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:00:51 smithi069 ceph-mon[21800]: rocksdb: Compression algorithms supported: 2024-04-24T13:00:51.105 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:00:51 smithi069 ceph-mon[21800]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-04-24T13:00:51.105 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:00:51 smithi069 ceph-mon[21800]: rocksdb: kZSTD supported: 0 2024-04-24T13:00:51.105 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:00:51 smithi069 ceph-mon[21800]: rocksdb: kXpressCompression supported: 0 2024-04-24T13:00:51.105 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:00:51 smithi069 ceph-mon[21800]: rocksdb: kLZ4HCCompression supported: 1 2024-04-24T13:00:51.105 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:00:51 smithi069 ceph-mon[21800]: rocksdb: kLZ4Compression supported: 1 2024-04-24T13:00:51.105 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:00:51 smithi069 ceph-mon[21800]: rocksdb: kBZip2Compression supported: 0 2024-04-24T13:00:51.105 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:00:51 smithi069 ceph-mon[21800]: rocksdb: kZlibCompression supported: 1 2024-04-24T13:00:51.105 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:00:51 smithi069 ceph-mon[21800]: rocksdb: kSnappyCompression supported: 1 2024-04-24T13:00:51.106 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:00:51 smithi069 ceph-mon[21800]: rocksdb: Fast CRC32 supported: Supported on x86 2024-04-24T13:00:51.106 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:00:51 smithi069 ceph-mon[21800]: rocksdb: [db/version_set.cc:4725] Recovering from manifest file: /var/lib/ceph/mon/ceph-smithi069/store.db/MANIFEST-000009 2024-04-24T13:00:51.106 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:00:51 smithi069 ceph-mon[21800]: rocksdb: [db/column_family.cc:597] --------------- Options for column family [default]: 2024-04-24T13:00:51.106 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:00:51 smithi069 ceph-mon[21800]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-04-24T13:00:51.106 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:00:51 smithi069 ceph-mon[21800]: rocksdb: Options.merge_operator: 2024-04-24T13:00:51.106 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:00:51 smithi069 ceph-mon[21800]: rocksdb: Options.compaction_filter: None 2024-04-24T13:00:51.106 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:00:51 smithi069 ceph-mon[21800]: rocksdb: Options.compaction_filter_factory: None 2024-04-24T13:00:51.106 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:00:51 smithi069 ceph-mon[21800]: rocksdb: Options.sst_partitioner_factory: None 2024-04-24T13:00:51.106 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:00:51 smithi069 ceph-mon[21800]: rocksdb: Options.memtable_factory: SkipListFactory 2024-04-24T13:00:51.106 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:00:51 smithi069 ceph-mon[21800]: rocksdb: Options.table_factory: BlockBasedTable 2024-04-24T13:00:51.106 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:00:51 smithi069 ceph-mon[21800]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55c9fbdf7d00) 2024-04-24T13:00:51.106 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: cache_index_and_filter_blocks: 1 2024-04-24T13:00:51.107 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-04-24T13:00:51.107 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-04-24T13:00:51.107 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: pin_top_level_index_and_filter: 1 2024-04-24T13:00:51.107 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: index_type: 0 2024-04-24T13:00:51.107 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: data_block_index_type: 0 2024-04-24T13:00:51.107 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: index_shortening: 1 2024-04-24T13:00:51.107 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: data_block_hash_table_util_ratio: 0.750000 2024-04-24T13:00:51.107 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: hash_index_allow_collision: 1 2024-04-24T13:00:51.107 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: checksum: 1 2024-04-24T13:00:51.107 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: no_block_cache: 0 2024-04-24T13:00:51.107 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: block_cache: 0x55c9fbe62170 2024-04-24T13:00:51.107 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: block_cache_name: BinnedLRUCache 2024-04-24T13:00:51.107 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: block_cache_options: 2024-04-24T13:00:51.107 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: capacity : 536870912 2024-04-24T13:00:51.107 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: num_shard_bits : 4 2024-04-24T13:00:51.108 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: strict_capacity_limit : 0 2024-04-24T13:00:51.108 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: high_pri_pool_ratio: 0.000 2024-04-24T13:00:51.108 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: block_cache_compressed: (nil) 2024-04-24T13:00:51.108 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: persistent_cache: (nil) 2024-04-24T13:00:51.108 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: block_size: 4096 2024-04-24T13:00:51.108 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: block_size_deviation: 10 2024-04-24T13:00:51.108 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: block_restart_interval: 16 2024-04-24T13:00:51.108 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: index_block_restart_interval: 1 2024-04-24T13:00:51.108 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: metadata_block_size: 4096 2024-04-24T13:00:51.108 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: partition_filters: 0 2024-04-24T13:00:51.108 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: use_delta_encoding: 1 2024-04-24T13:00:51.108 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: filter_policy: rocksdb.BuiltinBloomFilter 2024-04-24T13:00:51.108 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: whole_key_filtering: 1 2024-04-24T13:00:51.108 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: verify_compression: 0 2024-04-24T13:00:51.108 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: read_amp_bytes_per_bit: 0 2024-04-24T13:00:51.108 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: format_version: 4 2024-04-24T13:00:51.109 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: enable_index_compression: 1 2024-04-24T13:00:51.109 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: block_align: 0 2024-04-24T13:00:51.109 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:00:51 smithi069 ceph-mon[21800]: rocksdb: Options.write_buffer_size: 33554432 2024-04-24T13:00:51.109 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:00:51 smithi069 ceph-mon[21800]: rocksdb: Options.max_write_buffer_number: 2 2024-04-24T13:00:51.109 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:00:51 smithi069 ceph-mon[21800]: rocksdb: Options.compression: NoCompression 2024-04-24T13:00:51.109 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:00:51 smithi069 ceph-mon[21800]: rocksdb: Options.bottommost_compression: Disabled 2024-04-24T13:00:51.109 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:00:51 smithi069 ceph-mon[21800]: rocksdb: Options.prefix_extractor: nullptr 2024-04-24T13:00:51.109 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:00:51 smithi069 ceph-mon[21800]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-04-24T13:00:51.109 INFO:teuthology.orchestra.run.smithi069.stderr:Setting mon public_network to 172.21.0.0/20 2024-04-24T13:00:51.110 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:00:51 smithi069 ceph-mon[21800]: rocksdb: Options.num_levels: 7 2024-04-24T13:00:51.110 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:00:51 smithi069 ceph-mon[21800]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-04-24T13:00:51.110 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:00:51 smithi069 ceph-mon[21800]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-04-24T13:00:51.111 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:00:51 smithi069 ceph-mon[21800]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-04-24T13:00:51.111 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:00:51 smithi069 ceph-mon[21800]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-04-24T13:00:51.111 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:00:51 smithi069 ceph-mon[21800]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-04-24T13:00:51.111 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:00:51 smithi069 ceph-mon[21800]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-04-24T13:00:51.111 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:00:51 smithi069 ceph-mon[21800]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-04-24T13:00:51.111 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:00:51 smithi069 ceph-mon[21800]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-04-24T13:00:51.111 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:00:51 smithi069 ceph-mon[21800]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-04-24T13:00:51.111 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:00:51 smithi069 ceph-mon[21800]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-04-24T13:00:51.111 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:00:51 smithi069 ceph-mon[21800]: rocksdb: Options.compression_opts.window_bits: -14 2024-04-24T13:00:51.111 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:00:51 smithi069 ceph-mon[21800]: rocksdb: Options.compression_opts.level: 32767 2024-04-24T13:00:51.111 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:00:51 smithi069 ceph-mon[21800]: rocksdb: Options.compression_opts.strategy: 0 2024-04-24T13:00:51.111 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:00:51 smithi069 ceph-mon[21800]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-04-24T13:00:51.111 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:00:51 smithi069 ceph-mon[21800]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-04-24T13:00:51.111 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:00:51 smithi069 ceph-mon[21800]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-04-24T13:00:51.111 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:00:51 smithi069 ceph-mon[21800]: rocksdb: Options.compression_opts.enabled: false 2024-04-24T13:00:51.111 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:00:51 smithi069 ceph-mon[21800]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-04-24T13:00:51.112 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:00:51 smithi069 ceph-mon[21800]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-04-24T13:00:51.112 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:00:51 smithi069 ceph-mon[21800]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-04-24T13:00:51.112 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:00:51 smithi069 ceph-mon[21800]: rocksdb: Options.target_file_size_base: 67108864 2024-04-24T13:00:51.112 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:00:51 smithi069 ceph-mon[21800]: rocksdb: Options.target_file_size_multiplier: 1 2024-04-24T13:00:51.112 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:00:51 smithi069 ceph-mon[21800]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-04-24T13:00:51.112 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:00:51 smithi069 ceph-mon[21800]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-04-24T13:00:51.112 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:00:51 smithi069 ceph-mon[21800]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-04-24T13:00:51.112 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:00:51 smithi069 ceph-mon[21800]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-04-24T13:00:51.112 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:00:51 smithi069 ceph-mon[21800]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-04-24T13:00:51.112 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:00:51 smithi069 ceph-mon[21800]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-04-24T13:00:51.112 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:00:51 smithi069 ceph-mon[21800]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-04-24T13:00:51.113 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:00:51 smithi069 ceph-mon[21800]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-04-24T13:00:51.113 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:00:51 smithi069 ceph-mon[21800]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-04-24T13:00:51.113 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:00:51 smithi069 ceph-mon[21800]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-04-24T13:00:51.113 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:00:51 smithi069 ceph-mon[21800]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-04-24T13:00:51.113 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:00:51 smithi069 ceph-mon[21800]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-04-24T13:00:51.113 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:00:51 smithi069 ceph-mon[21800]: rocksdb: Options.arena_block_size: 4194304 2024-04-24T13:00:51.113 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:00:51 smithi069 ceph-mon[21800]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-04-24T13:00:51.113 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:00:51 smithi069 ceph-mon[21800]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-04-24T13:00:51.113 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:00:51 smithi069 ceph-mon[21800]: rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2024-04-24T13:00:51.113 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:00:51 smithi069 ceph-mon[21800]: rocksdb: Options.disable_auto_compactions: 0 2024-04-24T13:00:51.113 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:00:51 smithi069 ceph-mon[21800]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-04-24T13:00:51.113 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:00:51 smithi069 ceph-mon[21800]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-04-24T13:00:51.114 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:00:51 smithi069 ceph-mon[21800]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-04-24T13:00:51.114 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:00:51 smithi069 ceph-mon[21800]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-04-24T13:00:51.114 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:00:51 smithi069 ceph-mon[21800]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-04-24T13:00:51.114 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:00:51 smithi069 ceph-mon[21800]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-04-24T13:00:51.114 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:00:51 smithi069 ceph-mon[21800]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-04-24T13:00:51.114 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:00:51 smithi069 ceph-mon[21800]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-04-24T13:00:51.114 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:00:51 smithi069 ceph-mon[21800]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-04-24T13:00:51.114 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:00:51 smithi069 ceph-mon[21800]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-04-24T13:00:51.114 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:00:51 smithi069 ceph-mon[21800]: rocksdb: Options.table_properties_collectors: 2024-04-24T13:00:51.114 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:00:51 smithi069 ceph-mon[21800]: rocksdb: Options.inplace_update_support: 0 2024-04-24T13:00:51.114 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:00:51 smithi069 ceph-mon[21800]: rocksdb: Options.inplace_update_num_locks: 10000 2024-04-24T13:00:51.114 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:00:51 smithi069 ceph-mon[21800]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-04-24T13:00:51.114 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:00:51 smithi069 ceph-mon[21800]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-04-24T13:00:51.114 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:00:51 smithi069 ceph-mon[21800]: rocksdb: Options.memtable_huge_page_size: 0 2024-04-24T13:00:51.114 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:00:51 smithi069 ceph-mon[21800]: rocksdb: Options.bloom_locality: 0 2024-04-24T13:00:51.115 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:00:51 smithi069 ceph-mon[21800]: rocksdb: Options.max_successive_merges: 0 2024-04-24T13:00:51.115 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:00:51 smithi069 ceph-mon[21800]: rocksdb: Options.optimize_filters_for_hits: 0 2024-04-24T13:00:51.115 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:00:51 smithi069 ceph-mon[21800]: rocksdb: Options.paranoid_file_checks: 0 2024-04-24T13:00:51.115 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:00:51 smithi069 ceph-mon[21800]: rocksdb: Options.force_consistency_checks: 1 2024-04-24T13:00:51.115 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:00:51 smithi069 ceph-mon[21800]: rocksdb: Options.report_bg_io_stats: 0 2024-04-24T13:00:51.115 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:00:51 smithi069 ceph-mon[21800]: rocksdb: Options.ttl: 2592000 2024-04-24T13:00:51.115 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:00:51 smithi069 ceph-mon[21800]: rocksdb: Options.periodic_compaction_seconds: 0 2024-04-24T13:00:51.115 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:00:51 smithi069 ceph-mon[21800]: rocksdb: Options.enable_blob_files: false 2024-04-24T13:00:51.115 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:00:51 smithi069 ceph-mon[21800]: rocksdb: Options.min_blob_size: 0 2024-04-24T13:00:51.115 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:00:51 smithi069 ceph-mon[21800]: rocksdb: Options.blob_file_size: 268435456 2024-04-24T13:00:51.116 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:00:51 smithi069 ceph-mon[21800]: rocksdb: Options.blob_compression_type: NoCompression 2024-04-24T13:00:51.116 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:00:51 smithi069 ceph-mon[21800]: rocksdb: Options.enable_blob_garbage_collection: false 2024-04-24T13:00:51.116 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:00:51 smithi069 ceph-mon[21800]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-04-24T13:00:51.116 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:00:51 smithi069 ceph-mon[21800]: rocksdb: [db/version_set.cc:4773] Recovered from manifest file:/var/lib/ceph/mon/ceph-smithi069/store.db/MANIFEST-000009 succeeded,manifest_file_number is 9, next_file_number is 11, last_sequence is 5, log_number is 5,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-04-24T13:00:51.116 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:00:51 smithi069 ceph-mon[21800]: rocksdb: [db/version_set.cc:4782] Column family [default] (ID 0), log number is 5 2024-04-24T13:00:51.116 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:00:51 smithi069 ceph-mon[21800]: rocksdb: [db/version_set.cc:4083] Creating manifest 13 2024-04-24T13:00:51.116 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:00:51 smithi069 ceph-mon[21800]: rocksdb: EVENT_LOG_v1 {"time_micros": 1713963651043259, "job": 1, "event": "recovery_started", "wal_files": [10]} 2024-04-24T13:00:51.116 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:00:51 smithi069 ceph-mon[21800]: rocksdb: [db/db_impl/db_impl_open.cc:847] Recovering log #10 mode 2 2024-04-24T13:00:51.116 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:00:51 smithi069 ceph-mon[21800]: rocksdb: [table/block_based/filter_policy.cc:996] Using legacy Bloom filter with high (20) bits/key. Dramatic filter space and/or accuracy improvement is available with format_version>=5. 2024-04-24T13:00:51.116 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:00:51 smithi069 ceph-mon[21800]: rocksdb: EVENT_LOG_v1 {"time_micros": 1713963651043929, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 14, "file_size": 79387, "file_checksum": "", "file_checksum_func_name": "Unknown", "table_properties": {"data_size": 77651, "index_size": 224, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 581, "raw_key_size": 9884, "raw_average_key_size": 47, "raw_value_size": 71960, "raw_average_value_size": 344, "num_data_blocks": 10, "num_entries": 209, "num_deletions": 3, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "rocksdb.BuiltinBloomFilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; ", "creation_time": 1713963651, "oldest_key_time": 0, "file_creation_time": 0, "db_id": "4cf02008-90cb-493a-9670-fac56ea1d468", "db_session_id": "2XO3DVAU93V0AP7OENFO"}} 2024-04-24T13:00:51.116 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:00:51 smithi069 ceph-mon[21800]: rocksdb: [db/version_set.cc:4083] Creating manifest 15 2024-04-24T13:00:51.116 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:00:51 smithi069 ceph-mon[21800]: rocksdb: EVENT_LOG_v1 {"time_micros": 1713963651044272, "job": 1, "event": "recovery_finished"} 2024-04-24T13:00:51.116 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:00:51 smithi069 ceph-mon[21800]: rocksdb: [file/delete_scheduler.cc:73] Deleted file /var/lib/ceph/mon/ceph-smithi069/store.db/000010.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-04-24T13:00:51.116 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:00:51 smithi069 ceph-mon[21800]: rocksdb: [db/db_impl/db_impl_open.cc:1701] SstFileManager instance 0x55c9fbe48a80 2024-04-24T13:00:51.117 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:00:51 smithi069 ceph-mon[21800]: rocksdb: DB pointer 0x55c9fbe58000 2024-04-24T13:00:51.117 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:00:51 smithi069 ceph-mon[21800]: rocksdb: [db/db_impl/db_impl.cc:902] ------- DUMPING STATS ------- 2024-04-24T13:00:51.117 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:00:51 smithi069 ceph-mon[21800]: rocksdb: [db/db_impl/db_impl.cc:903] 2024-04-24T13:00:51.117 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: ** DB Stats ** 2024-04-24T13:00:51.117 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-24T13:00:51.117 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-04-24T13:00:51.117 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-04-24T13:00:51.117 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-04-24T13:00:51.117 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-04-24T13:00:51.117 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2024-04-24T13:00:51.117 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-04-24T13:00:51.117 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: 2024-04-24T13:00:51.118 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: ** Compaction Stats [default] ** 2024-04-24T13:00:51.118 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-04-24T13:00:51.118 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-24T13:00:51.118 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: L0 2/0 79.32 KB 0.5 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 166.8 0.00 0.00 1 0.000 0 0 2024-04-24T13:00:51.118 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: Sum 2/0 79.32 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 166.8 0.00 0.00 1 0.000 0 0 2024-04-24T13:00:51.118 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 166.8 0.00 0.00 1 0.000 0 0 2024-04-24T13:00:51.118 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: 2024-04-24T13:00:51.118 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: ** Compaction Stats [default] ** 2024-04-24T13:00:51.118 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-04-24T13:00:51.118 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-24T13:00:51.118 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 166.8 0.00 0.00 1 0.000 0 0 2024-04-24T13:00:51.118 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-24T13:00:51.118 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-04-24T13:00:51.119 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-04-24T13:00:51.119 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-04-24T13:00:51.119 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-04-24T13:00:51.119 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: AddFile(Keys): cumulative 0, interval 0 2024-04-24T13:00:51.119 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: Cumulative compaction: 0.00 GB write, 25.49 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-24T13:00:51.119 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: Interval compaction: 0.00 GB write, 25.49 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-24T13:00:51.119 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-04-24T13:00:51.119 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: 2024-04-24T13:00:51.119 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: ** File Read Latency Histogram By Level [default] ** 2024-04-24T13:00:51.119 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: 2024-04-24T13:00:51.119 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: ** Compaction Stats [default] ** 2024-04-24T13:00:51.119 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-04-24T13:00:51.119 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-24T13:00:51.119 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: L0 2/0 79.32 KB 0.5 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 166.8 0.00 0.00 1 0.000 0 0 2024-04-24T13:00:51.120 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: Sum 2/0 79.32 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 166.8 0.00 0.00 1 0.000 0 0 2024-04-24T13:00:51.120 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2024-04-24T13:00:51.120 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: 2024-04-24T13:00:51.120 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: ** Compaction Stats [default] ** 2024-04-24T13:00:51.120 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-04-24T13:00:51.120 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-24T13:00:51.120 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 166.8 0.00 0.00 1 0.000 0 0 2024-04-24T13:00:51.120 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-04-24T13:00:51.120 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-04-24T13:00:51.120 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-04-24T13:00:51.121 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-04-24T13:00:51.121 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-04-24T13:00:51.121 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: AddFile(Keys): cumulative 0, interval 0 2024-04-24T13:00:51.121 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: Cumulative compaction: 0.00 GB write, 25.23 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-24T13:00:51.121 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-24T13:00:51.121 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-04-24T13:00:51.121 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: 2024-04-24T13:00:51.121 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: ** File Read Latency Histogram By Level [default] ** 2024-04-24T13:00:51.121 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:00:51 smithi069 ceph-mon[21800]: starting mon.smithi069 rank 0 at public addrs [v2:172.21.15.69:3300/0,v1:172.21.15.69:6789/0] at bind addrs [v2:172.21.15.69:3300/0,v1:172.21.15.69:6789/0] mon_data /var/lib/ceph/mon/ceph-smithi069 fsid 9150213e-023a-11ef-bc93-c7b262605968 2024-04-24T13:00:51.121 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:00:51 smithi069 ceph-mon[21800]: mon.smithi069@-1(???) e1 preinit fsid 9150213e-023a-11ef-bc93-c7b262605968 2024-04-24T13:00:51.121 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:00:51 smithi069 ceph-mon[21800]: mon.smithi069@-1(???).mds e1 new map 2024-04-24T13:00:51.121 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:00:51 smithi069 ceph-mon[21800]: mon.smithi069@-1(???).mds e1 print_map 2024-04-24T13:00:51.122 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: e1 2024-04-24T13:00:51.122 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-04-24T13:00:51.122 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-04-24T13:00:51.122 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: legacy client fscid: -1 2024-04-24T13:00:51.122 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: 2024-04-24T13:00:51.122 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout: No filesystems configured 2024-04-24T13:00:51.122 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:00:51 smithi069 ceph-mon[21800]: mon.smithi069@-1(???).osd e1 crush map has features 3314932999778484224, adjusting msgr requires 2024-04-24T13:00:51.122 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:00:51 smithi069 ceph-mon[21800]: mon.smithi069@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-04-24T13:00:51.122 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:00:51 smithi069 ceph-mon[21800]: mon.smithi069@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-04-24T13:00:51.122 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:00:51 smithi069 ceph-mon[21800]: mon.smithi069@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-04-24T13:00:51.122 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:00:51 smithi069 ceph-mon[21800]: mon.smithi069@-1(???).paxosservice(auth 1..2) refresh upgraded, format 0 -> 3 2024-04-24T13:00:51.123 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:00:51 smithi069 ceph-mon[21800]: expand_channel_meta expand map: {default=false} 2024-04-24T13:00:51.123 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:00:51 smithi069 ceph-mon[21800]: expand_channel_meta from 'false' to 'false' 2024-04-24T13:00:51.123 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:00:51 smithi069 ceph-mon[21800]: expand_channel_meta expanded map: {default=false} 2024-04-24T13:00:51.123 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:00:51 smithi069 ceph-mon[21800]: expand_channel_meta expand map: {default=info} 2024-04-24T13:00:51.123 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:00:51 smithi069 ceph-mon[21800]: expand_channel_meta from 'info' to 'info' 2024-04-24T13:00:51.123 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:00:51 smithi069 ceph-mon[21800]: expand_channel_meta expanded map: {default=info} 2024-04-24T13:00:51.123 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:00:51 smithi069 ceph-mon[21800]: expand_channel_meta expand map: {default=daemon} 2024-04-24T13:00:51.123 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:00:51 smithi069 ceph-mon[21800]: expand_channel_meta from 'daemon' to 'daemon' 2024-04-24T13:00:51.123 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:00:51 smithi069 ceph-mon[21800]: expand_channel_meta expanded map: {default=daemon} 2024-04-24T13:00:51.123 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:00:51 smithi069 ceph-mon[21800]: expand_channel_meta expand map: {default=debug} 2024-04-24T13:00:51.127 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:00:51 smithi069 ceph-mon[21800]: expand_channel_meta from 'debug' to 'debug' 2024-04-24T13:00:51.127 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:00:51 smithi069 ceph-mon[21800]: expand_channel_meta expanded map: {default=debug} 2024-04-24T13:00:51.127 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:00:51 smithi069 bash[21781]: 82aada682e0492920095b94da619855b8ceb3ca28c06247cbdccba51b632f64b 2024-04-24T13:00:51.127 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:00:51 smithi069 systemd[1]: Started Ceph mon.smithi069 for 9150213e-023a-11ef-bc93-c7b262605968. 2024-04-24T13:00:51.349 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:00:51 smithi069 ceph-mon[21800]: mon.smithi069 is new leader, mons smithi069 in quorum (ranks 0) 2024-04-24T13:00:51.529 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:00:51 smithi069 ceph-mon[21800]: monmap e1: 1 mons at {smithi069=[v2:172.21.15.69:3300/0,v1:172.21.15.69:6789/0]} 2024-04-24T13:00:51.529 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:00:51 smithi069 ceph-mon[21800]: fsmap 2024-04-24T13:00:51.529 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:00:51 smithi069 ceph-mon[21800]: osdmap e1: 0 total, 0 up, 0 in 2024-04-24T13:00:51.529 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:00:51 smithi069 ceph-mon[21800]: mgrmap e1: no daemons active 2024-04-24T13:00:52.095 INFO:teuthology.orchestra.run.smithi069.stderr:Wrote config to /etc/ceph/ceph.conf 2024-04-24T13:00:52.095 INFO:teuthology.orchestra.run.smithi069.stderr:Wrote keyring to /etc/ceph/ceph.client.admin.keyring 2024-04-24T13:00:52.095 INFO:teuthology.orchestra.run.smithi069.stderr:Creating mgr... 2024-04-24T13:00:52.095 INFO:teuthology.orchestra.run.smithi069.stderr:Verifying port 9283 ... 2024-04-24T13:00:52.269 INFO:teuthology.orchestra.run.smithi069.stderr:systemctl: Failed to reset failed state of unit ceph-9150213e-023a-11ef-bc93-c7b262605968@mgr.smithi069.vspncx.service: Unit ceph-9150213e-023a-11ef-bc93-c7b262605968@mgr.smithi069.vspncx.service not loaded. 2024-04-24T13:00:52.275 INFO:teuthology.orchestra.run.smithi069.stderr:systemctl: Created symlink /etc/systemd/system/ceph-9150213e-023a-11ef-bc93-c7b262605968.target.wants/ceph-9150213e-023a-11ef-bc93-c7b262605968@mgr.smithi069.vspncx.service → /etc/systemd/system/ceph-9150213e-023a-11ef-bc93-c7b262605968@.service. 2024-04-24T13:00:52.766 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:00:52 smithi069 ceph-mon[21800]: from='client.? 172.21.15.69:0/1957563003' entity='client.admin' 2024-04-24T13:00:52.958 INFO:teuthology.orchestra.run.smithi069.stderr:systemctl: disabled 2024-04-24T13:00:52.965 INFO:teuthology.orchestra.run.smithi069.stderr:systemctl: inactive 2024-04-24T13:00:52.966 INFO:teuthology.orchestra.run.smithi069.stderr:firewalld.service is not enabled 2024-04-24T13:00:52.966 INFO:teuthology.orchestra.run.smithi069.stderr:Not possible to enable service . firewalld.service is not available 2024-04-24T13:00:52.973 INFO:teuthology.orchestra.run.smithi069.stderr:systemctl: disabled 2024-04-24T13:00:52.980 INFO:teuthology.orchestra.run.smithi069.stderr:systemctl: inactive 2024-04-24T13:00:52.981 INFO:teuthology.orchestra.run.smithi069.stderr:firewalld.service is not enabled 2024-04-24T13:00:52.981 INFO:teuthology.orchestra.run.smithi069.stderr:Not possible to open ports <[9283]>. firewalld.service is not available 2024-04-24T13:00:52.981 INFO:teuthology.orchestra.run.smithi069.stderr:Waiting for mgr to start... 2024-04-24T13:00:52.982 INFO:teuthology.orchestra.run.smithi069.stderr:Waiting for mgr... 2024-04-24T13:00:53.478 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: 2024-04-24T13:00:53.479 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: { 2024-04-24T13:00:53.479 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "fsid": "9150213e-023a-11ef-bc93-c7b262605968", 2024-04-24T13:00:53.479 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "health": { 2024-04-24T13:00:53.479 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "status": "HEALTH_OK", 2024-04-24T13:00:53.479 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "checks": {}, 2024-04-24T13:00:53.479 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "mutes": [] 2024-04-24T13:00:53.479 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: }, 2024-04-24T13:00:53.479 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "election_epoch": 5, 2024-04-24T13:00:53.479 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "quorum": [ 2024-04-24T13:00:53.479 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: 0 2024-04-24T13:00:53.480 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: ], 2024-04-24T13:00:53.480 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "quorum_names": [ 2024-04-24T13:00:53.480 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "smithi069" 2024-04-24T13:00:53.480 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: ], 2024-04-24T13:00:53.480 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "quorum_age": 2, 2024-04-24T13:00:53.480 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "monmap": { 2024-04-24T13:00:53.480 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "epoch": 1, 2024-04-24T13:00:53.480 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "min_mon_release_name": "quincy", 2024-04-24T13:00:53.480 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "num_mons": 1 2024-04-24T13:00:53.480 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: }, 2024-04-24T13:00:53.480 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "osdmap": { 2024-04-24T13:00:53.480 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "epoch": 1, 2024-04-24T13:00:53.480 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "num_osds": 0, 2024-04-24T13:00:53.480 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "num_up_osds": 0, 2024-04-24T13:00:53.481 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "osd_up_since": 0, 2024-04-24T13:00:53.481 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "num_in_osds": 0, 2024-04-24T13:00:53.484 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "osd_in_since": 0, 2024-04-24T13:00:53.484 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "num_remapped_pgs": 0 2024-04-24T13:00:53.484 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: }, 2024-04-24T13:00:53.485 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "pgmap": { 2024-04-24T13:00:53.485 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "pgs_by_state": [], 2024-04-24T13:00:53.485 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "num_pgs": 0, 2024-04-24T13:00:53.485 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "num_pools": 0, 2024-04-24T13:00:53.485 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "num_objects": 0, 2024-04-24T13:00:53.485 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "data_bytes": 0, 2024-04-24T13:00:53.485 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "bytes_used": 0, 2024-04-24T13:00:53.485 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "bytes_avail": 0, 2024-04-24T13:00:53.486 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "bytes_total": 0 2024-04-24T13:00:53.486 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: }, 2024-04-24T13:00:53.486 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "fsmap": { 2024-04-24T13:00:53.486 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "epoch": 1, 2024-04-24T13:00:53.486 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "by_rank": [], 2024-04-24T13:00:53.486 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "up:standby": 0 2024-04-24T13:00:53.486 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: }, 2024-04-24T13:00:53.486 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "mgrmap": { 2024-04-24T13:00:53.486 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "available": false, 2024-04-24T13:00:53.486 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "num_standbys": 0, 2024-04-24T13:00:53.486 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "modules": [ 2024-04-24T13:00:53.486 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "iostat", 2024-04-24T13:00:53.487 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "nfs", 2024-04-24T13:00:53.487 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "restful" 2024-04-24T13:00:53.487 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: ], 2024-04-24T13:00:53.487 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "services": {} 2024-04-24T13:00:53.487 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: }, 2024-04-24T13:00:53.487 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "servicemap": { 2024-04-24T13:00:53.487 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "epoch": 1, 2024-04-24T13:00:53.487 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "modified": "2024-04-24T13:00:46.137228+0000", 2024-04-24T13:00:53.487 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "services": {} 2024-04-24T13:00:53.487 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: }, 2024-04-24T13:00:53.487 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "progress_events": {} 2024-04-24T13:00:53.487 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: } 2024-04-24T13:00:53.850 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:00:53 smithi069 ceph-mon[21800]: from='client.? 172.21.15.69:0/1111793222' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-04-24T13:00:54.438 INFO:teuthology.orchestra.run.smithi069.stderr:mgr not available, waiting (1/15)... 2024-04-24T13:00:56.955 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: 2024-04-24T13:00:56.955 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: { 2024-04-24T13:00:56.955 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "fsid": "9150213e-023a-11ef-bc93-c7b262605968", 2024-04-24T13:00:56.956 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "health": { 2024-04-24T13:00:56.956 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "status": "HEALTH_OK", 2024-04-24T13:00:56.956 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "checks": {}, 2024-04-24T13:00:56.956 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "mutes": [] 2024-04-24T13:00:56.956 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: }, 2024-04-24T13:00:56.956 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "election_epoch": 5, 2024-04-24T13:00:56.956 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "quorum": [ 2024-04-24T13:00:56.956 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: 0 2024-04-24T13:00:56.956 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: ], 2024-04-24T13:00:56.956 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "quorum_names": [ 2024-04-24T13:00:56.957 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "smithi069" 2024-04-24T13:00:56.957 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: ], 2024-04-24T13:00:56.957 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "quorum_age": 5, 2024-04-24T13:00:56.957 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "monmap": { 2024-04-24T13:00:56.957 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "epoch": 1, 2024-04-24T13:00:56.957 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "min_mon_release_name": "quincy", 2024-04-24T13:00:56.957 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "num_mons": 1 2024-04-24T13:00:56.957 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: }, 2024-04-24T13:00:56.957 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "osdmap": { 2024-04-24T13:00:56.957 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "epoch": 1, 2024-04-24T13:00:56.957 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "num_osds": 0, 2024-04-24T13:00:56.957 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "num_up_osds": 0, 2024-04-24T13:00:56.957 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "osd_up_since": 0, 2024-04-24T13:00:56.957 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "num_in_osds": 0, 2024-04-24T13:00:56.957 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "osd_in_since": 0, 2024-04-24T13:00:56.957 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "num_remapped_pgs": 0 2024-04-24T13:00:56.958 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: }, 2024-04-24T13:00:56.958 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "pgmap": { 2024-04-24T13:00:56.958 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "pgs_by_state": [], 2024-04-24T13:00:56.958 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "num_pgs": 0, 2024-04-24T13:00:56.958 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "num_pools": 0, 2024-04-24T13:00:56.958 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "num_objects": 0, 2024-04-24T13:00:56.958 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "data_bytes": 0, 2024-04-24T13:00:56.958 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "bytes_used": 0, 2024-04-24T13:00:56.958 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "bytes_avail": 0, 2024-04-24T13:00:56.958 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "bytes_total": 0 2024-04-24T13:00:56.958 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: }, 2024-04-24T13:00:56.958 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "fsmap": { 2024-04-24T13:00:56.958 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "epoch": 1, 2024-04-24T13:00:56.958 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "by_rank": [], 2024-04-24T13:00:56.958 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "up:standby": 0 2024-04-24T13:00:56.959 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: }, 2024-04-24T13:00:56.959 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "mgrmap": { 2024-04-24T13:00:56.959 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "available": false, 2024-04-24T13:00:56.959 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "num_standbys": 0, 2024-04-24T13:00:56.959 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "modules": [ 2024-04-24T13:00:56.959 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "iostat", 2024-04-24T13:00:56.959 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "nfs", 2024-04-24T13:00:56.959 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "restful" 2024-04-24T13:00:56.959 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: ], 2024-04-24T13:00:56.959 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "services": {} 2024-04-24T13:00:56.959 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: }, 2024-04-24T13:00:56.959 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "servicemap": { 2024-04-24T13:00:56.959 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "epoch": 1, 2024-04-24T13:00:56.959 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "modified": "2024-04-24T13:00:46.137228+0000", 2024-04-24T13:00:56.959 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "services": {} 2024-04-24T13:00:56.960 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: }, 2024-04-24T13:00:56.960 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "progress_events": {} 2024-04-24T13:00:56.960 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: } 2024-04-24T13:00:57.153 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:00:57 smithi069 ceph-mon[21800]: from='client.? 172.21.15.69:0/3596462946' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-04-24T13:00:57.485 INFO:teuthology.orchestra.run.smithi069.stderr:mgr not available, waiting (2/15)... 2024-04-24T13:00:59.100 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:00:58 smithi069 ceph-mon[21800]: Activating manager daemon smithi069.vspncx 2024-04-24T13:00:59.101 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:00:58 smithi069 ceph-mon[21800]: mgrmap e2: smithi069.vspncx(active, starting, since 0.00286933s) 2024-04-24T13:00:59.101 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:00:58 smithi069 ceph-mon[21800]: from='mgr.14100 172.21.15.69:0/1242657614' entity='mgr.smithi069.vspncx' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-24T13:00:59.101 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:00:58 smithi069 ceph-mon[21800]: from='mgr.14100 172.21.15.69:0/1242657614' entity='mgr.smithi069.vspncx' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-24T13:00:59.101 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:00:58 smithi069 ceph-mon[21800]: from='mgr.14100 172.21.15.69:0/1242657614' entity='mgr.smithi069.vspncx' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-24T13:00:59.101 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:00:58 smithi069 ceph-mon[21800]: from='mgr.14100 172.21.15.69:0/1242657614' entity='mgr.smithi069.vspncx' cmd=[{"prefix": "mon metadata", "id": "smithi069"}]: dispatch 2024-04-24T13:00:59.101 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:00:58 smithi069 ceph-mon[21800]: from='mgr.14100 172.21.15.69:0/1242657614' entity='mgr.smithi069.vspncx' cmd=[{"prefix": "mgr metadata", "who": "smithi069.vspncx", "id": "smithi069.vspncx"}]: dispatch 2024-04-24T13:00:59.101 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:00:58 smithi069 ceph-mon[21800]: Manager daemon smithi069.vspncx is now available 2024-04-24T13:00:59.101 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:00:58 smithi069 ceph-mon[21800]: from='mgr.14100 172.21.15.69:0/1242657614' entity='mgr.smithi069.vspncx' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi069.vspncx/mirror_snapshot_schedule"}]: dispatch 2024-04-24T13:00:59.101 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:00:58 smithi069 ceph-mon[21800]: from='mgr.14100 172.21.15.69:0/1242657614' entity='mgr.smithi069.vspncx' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi069.vspncx/trash_purge_schedule"}]: dispatch 2024-04-24T13:00:59.101 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:00:58 smithi069 ceph-mon[21800]: from='mgr.14100 172.21.15.69:0/1242657614' entity='mgr.smithi069.vspncx' 2024-04-24T13:00:59.101 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:00:58 smithi069 ceph-mon[21800]: from='mgr.14100 172.21.15.69:0/1242657614' entity='mgr.smithi069.vspncx' 2024-04-24T13:00:59.102 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:00:58 smithi069 ceph-mon[21800]: from='mgr.14100 172.21.15.69:0/1242657614' entity='mgr.smithi069.vspncx' 2024-04-24T13:01:00.158 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: 2024-04-24T13:01:00.159 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: { 2024-04-24T13:01:00.159 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "fsid": "9150213e-023a-11ef-bc93-c7b262605968", 2024-04-24T13:01:00.159 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "health": { 2024-04-24T13:01:00.159 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "status": "HEALTH_OK", 2024-04-24T13:01:00.159 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "checks": {}, 2024-04-24T13:01:00.159 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "mutes": [] 2024-04-24T13:01:00.159 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: }, 2024-04-24T13:01:00.159 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "election_epoch": 5, 2024-04-24T13:01:00.159 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "quorum": [ 2024-04-24T13:01:00.159 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: 0 2024-04-24T13:01:00.159 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: ], 2024-04-24T13:01:00.159 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "quorum_names": [ 2024-04-24T13:01:00.159 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "smithi069" 2024-04-24T13:01:00.159 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: ], 2024-04-24T13:01:00.159 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "quorum_age": 9, 2024-04-24T13:01:00.160 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "monmap": { 2024-04-24T13:01:00.160 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "epoch": 1, 2024-04-24T13:01:00.160 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "min_mon_release_name": "quincy", 2024-04-24T13:01:00.160 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "num_mons": 1 2024-04-24T13:01:00.160 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: }, 2024-04-24T13:01:00.160 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "osdmap": { 2024-04-24T13:01:00.160 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "epoch": 1, 2024-04-24T13:01:00.160 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "num_osds": 0, 2024-04-24T13:01:00.160 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "num_up_osds": 0, 2024-04-24T13:01:00.160 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "osd_up_since": 0, 2024-04-24T13:01:00.160 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "num_in_osds": 0, 2024-04-24T13:01:00.160 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "osd_in_since": 0, 2024-04-24T13:01:00.160 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "num_remapped_pgs": 0 2024-04-24T13:01:00.160 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: }, 2024-04-24T13:01:00.160 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "pgmap": { 2024-04-24T13:01:00.160 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "pgs_by_state": [], 2024-04-24T13:01:00.161 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "num_pgs": 0, 2024-04-24T13:01:00.161 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "num_pools": 0, 2024-04-24T13:01:00.161 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "num_objects": 0, 2024-04-24T13:01:00.161 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "data_bytes": 0, 2024-04-24T13:01:00.161 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "bytes_used": 0, 2024-04-24T13:01:00.161 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "bytes_avail": 0, 2024-04-24T13:01:00.161 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "bytes_total": 0 2024-04-24T13:01:00.161 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: }, 2024-04-24T13:01:00.161 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "fsmap": { 2024-04-24T13:01:00.161 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "epoch": 1, 2024-04-24T13:01:00.161 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "by_rank": [], 2024-04-24T13:01:00.162 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "up:standby": 0 2024-04-24T13:01:00.162 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: }, 2024-04-24T13:01:00.162 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "mgrmap": { 2024-04-24T13:01:00.162 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "available": true, 2024-04-24T13:01:00.162 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "num_standbys": 0, 2024-04-24T13:01:00.162 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "modules": [ 2024-04-24T13:01:00.162 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "iostat", 2024-04-24T13:01:00.162 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "nfs", 2024-04-24T13:01:00.162 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "restful" 2024-04-24T13:01:00.162 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: ], 2024-04-24T13:01:00.162 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "services": {} 2024-04-24T13:01:00.162 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: }, 2024-04-24T13:01:00.162 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "servicemap": { 2024-04-24T13:01:00.162 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "epoch": 1, 2024-04-24T13:01:00.163 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "modified": "2024-04-24T13:00:46.137228+0000", 2024-04-24T13:01:00.163 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "services": {} 2024-04-24T13:01:00.163 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: }, 2024-04-24T13:01:00.163 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "progress_events": {} 2024-04-24T13:01:00.163 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: } 2024-04-24T13:01:00.735 INFO:teuthology.orchestra.run.smithi069.stderr:mgr is available 2024-04-24T13:01:01.085 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:01:00 smithi069 ceph-mon[21800]: mgrmap e3: smithi069.vspncx(active, since 1.00579s) 2024-04-24T13:01:01.085 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:01:00 smithi069 ceph-mon[21800]: from='client.? 172.21.15.69:0/3828011809' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-04-24T13:01:01.360 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: 2024-04-24T13:01:01.360 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: [global] 2024-04-24T13:01:01.360 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: fsid = 9150213e-023a-11ef-bc93-c7b262605968 2024-04-24T13:01:01.360 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: mon_osd_allow_pg_remap = true 2024-04-24T13:01:01.360 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: mon_osd_allow_primary_affinity = true 2024-04-24T13:01:01.360 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: mon_warn_on_no_sortbitwise = false 2024-04-24T13:01:01.360 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: osd_crush_chooseleaf_type = 0 2024-04-24T13:01:01.361 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: 2024-04-24T13:01:01.361 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: [mgr] 2024-04-24T13:01:01.361 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: mgr/telemetry/nag = false 2024-04-24T13:01:01.361 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: 2024-04-24T13:01:01.361 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: [osd] 2024-04-24T13:01:01.361 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: osd_map_max_advance = 10 2024-04-24T13:01:01.361 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: osd_sloppy_crc = true 2024-04-24T13:01:01.958 INFO:teuthology.orchestra.run.smithi069.stderr:Enabling cephadm module... 2024-04-24T13:01:02.106 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:01:01 smithi069 ceph-mon[21800]: mgrmap e4: smithi069.vspncx(active, since 2s) 2024-04-24T13:01:02.106 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:01:01 smithi069 ceph-mon[21800]: from='client.? 172.21.15.69:0/1491653036' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2024-04-24T13:01:03.141 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:01:02 smithi069 ceph-mon[21800]: from='client.? 172.21.15.69:0/1416810030' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "cephadm"}]: dispatch 2024-04-24T13:01:04.100 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:01:03 smithi069 ceph-mon[21800]: from='client.? 172.21.15.69:0/1416810030' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "cephadm"}]': finished 2024-04-24T13:01:04.100 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:01:03 smithi069 ceph-mon[21800]: mgrmap e5: smithi069.vspncx(active, since 4s) 2024-04-24T13:01:04.148 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: { 2024-04-24T13:01:04.148 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "epoch": 5, 2024-04-24T13:01:04.148 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "available": true, 2024-04-24T13:01:04.148 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "active_name": "smithi069.vspncx", 2024-04-24T13:01:04.149 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "num_standby": 0 2024-04-24T13:01:04.149 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: } 2024-04-24T13:01:04.679 INFO:teuthology.orchestra.run.smithi069.stderr:Waiting for the mgr to restart... 2024-04-24T13:01:04.679 INFO:teuthology.orchestra.run.smithi069.stderr:Waiting for mgr epoch 5... 2024-04-24T13:01:05.087 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:01:04 smithi069 ceph-mon[21800]: from='client.? 172.21.15.69:0/3326312686' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2024-04-24T13:01:09.100 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:01:08 smithi069 ceph-mon[21800]: Active manager daemon smithi069.vspncx restarted 2024-04-24T13:01:09.101 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:01:08 smithi069 ceph-mon[21800]: Activating manager daemon smithi069.vspncx 2024-04-24T13:01:09.101 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:01:08 smithi069 ceph-mon[21800]: osdmap e2: 0 total, 0 up, 0 in 2024-04-24T13:01:09.892 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: { 2024-04-24T13:01:09.892 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "mgrmap_epoch": 7, 2024-04-24T13:01:09.892 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "initialized": true 2024-04-24T13:01:09.892 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: } 2024-04-24T13:01:10.195 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:01:09 smithi069 ceph-mon[21800]: mgrmap e6: smithi069.vspncx(active, starting, since 0.0524375s) 2024-04-24T13:01:10.195 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:01:09 smithi069 ceph-mon[21800]: from='mgr.14118 172.21.15.69:0/4119475000' entity='mgr.smithi069.vspncx' cmd=[{"prefix": "mon metadata", "id": "smithi069"}]: dispatch 2024-04-24T13:01:10.195 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:01:09 smithi069 ceph-mon[21800]: from='mgr.14118 172.21.15.69:0/4119475000' entity='mgr.smithi069.vspncx' cmd=[{"prefix": "mgr metadata", "who": "smithi069.vspncx", "id": "smithi069.vspncx"}]: dispatch 2024-04-24T13:01:10.195 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:01:09 smithi069 ceph-mon[21800]: from='mgr.14118 172.21.15.69:0/4119475000' entity='mgr.smithi069.vspncx' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-24T13:01:10.195 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:01:09 smithi069 ceph-mon[21800]: from='mgr.14118 172.21.15.69:0/4119475000' entity='mgr.smithi069.vspncx' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-24T13:01:10.196 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:01:09 smithi069 ceph-mon[21800]: from='mgr.14118 172.21.15.69:0/4119475000' entity='mgr.smithi069.vspncx' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-24T13:01:10.196 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:01:09 smithi069 ceph-mon[21800]: Manager daemon smithi069.vspncx is now available 2024-04-24T13:01:10.196 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:01:09 smithi069 ceph-mon[21800]: from='mgr.14118 172.21.15.69:0/4119475000' entity='mgr.smithi069.vspncx' 2024-04-24T13:01:10.196 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:01:09 smithi069 ceph-mon[21800]: from='mgr.14118 172.21.15.69:0/4119475000' entity='mgr.smithi069.vspncx' 2024-04-24T13:01:10.196 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:01:09 smithi069 ceph-mon[21800]: from='mgr.14118 172.21.15.69:0/4119475000' entity='mgr.smithi069.vspncx' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-24T13:01:10.196 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:01:09 smithi069 ceph-mon[21800]: from='mgr.14118 172.21.15.69:0/4119475000' entity='mgr.smithi069.vspncx' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-24T13:01:10.196 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:01:09 smithi069 ceph-mon[21800]: from='mgr.14118 172.21.15.69:0/4119475000' entity='mgr.smithi069.vspncx' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-24T13:01:10.196 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:01:09 smithi069 ceph-mon[21800]: from='mgr.14118 172.21.15.69:0/4119475000' entity='mgr.smithi069.vspncx' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi069.vspncx/mirror_snapshot_schedule"}]: dispatch 2024-04-24T13:01:10.196 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:01:09 smithi069 ceph-mon[21800]: from='mgr.14118 172.21.15.69:0/4119475000' entity='mgr.smithi069.vspncx' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi069.vspncx/trash_purge_schedule"}]: dispatch 2024-04-24T13:01:10.409 INFO:teuthology.orchestra.run.smithi069.stderr:mgr epoch 5 is available 2024-04-24T13:01:10.409 INFO:teuthology.orchestra.run.smithi069.stderr:Setting orchestrator backend to cephadm... 2024-04-24T13:01:11.101 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:01:10 smithi069 ceph-mon[21800]: from='client.14122 -' entity='client.admin' cmd=[{"prefix": "get_command_descriptions"}]: dispatch 2024-04-24T13:01:11.101 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:01:10 smithi069 ceph-mon[21800]: mgrmap e7: smithi069.vspncx(active, since 1.05724s) 2024-04-24T13:01:11.101 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:01:10 smithi069 ceph-mon[21800]: from='client.14122 -' entity='client.admin' cmd=[{"prefix": "mgr_status"}]: dispatch 2024-04-24T13:01:11.101 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:01:10 smithi069 ceph-mon[21800]: from='mgr.14118 172.21.15.69:0/4119475000' entity='mgr.smithi069.vspncx' 2024-04-24T13:01:11.101 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:01:10 smithi069 ceph-mon[21800]: from='mgr.14118 172.21.15.69:0/4119475000' entity='mgr.smithi069.vspncx' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-24T13:01:12.097 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:01:11 smithi069 ceph-mon[21800]: [24/Apr/2024:13:01:10] ENGINE Bus STARTING 2024-04-24T13:01:12.097 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:01:11 smithi069 ceph-mon[21800]: [24/Apr/2024:13:01:10] ENGINE Serving on https://172.21.15.69:7150 2024-04-24T13:01:12.097 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:01:11 smithi069 ceph-mon[21800]: [24/Apr/2024:13:01:10] ENGINE Bus STARTED 2024-04-24T13:01:12.097 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:01:11 smithi069 ceph-mon[21800]: from='mgr.14118 172.21.15.69:0/4119475000' entity='mgr.smithi069.vspncx' 2024-04-24T13:01:12.097 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:01:11 smithi069 ceph-mon[21800]: from='mgr.14118 172.21.15.69:0/4119475000' entity='mgr.smithi069.vspncx' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-24T13:01:12.097 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:01:11 smithi069 ceph-mon[21800]: mgrmap e8: smithi069.vspncx(active, since 2s) 2024-04-24T13:01:12.098 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: value unchanged 2024-04-24T13:01:12.653 INFO:teuthology.orchestra.run.smithi069.stderr:Generating ssh key... 2024-04-24T13:01:12.938 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:01:12 smithi069 ceph-mon[21800]: from='client.14130 -' entity='client.admin' cmd=[{"prefix": "orch set backend", "module_name": "cephadm", "target": ["mon-mgr", ""]}]: dispatch 2024-04-24T13:01:14.032 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:01:13 smithi069 ceph-mon[21800]: from='client.14132 -' entity='client.admin' cmd=[{"prefix": "cephadm set-user", "user": "root", "target": ["mon-mgr", ""]}]: dispatch 2024-04-24T13:01:14.032 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:01:13 smithi069 ceph-mon[21800]: from='mgr.14118 172.21.15.69:0/4119475000' entity='mgr.smithi069.vspncx' 2024-04-24T13:01:14.032 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:01:13 smithi069 ceph-mon[21800]: from='mgr.14118 172.21.15.69:0/4119475000' entity='mgr.smithi069.vspncx' 2024-04-24T13:01:14.638 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDT1MvGN6Xh22oyDaKzv/wFONtnFo7QGIwoRv/R3GhL/LbR7sNIvtPOygfwyXzIVi1oaeZVPkCxeu+UgGAqPUIqi+l9wDhDaVNhdaW2xhsUpqFFsLu5Ed6dgZ7QZK8itZT6aptwsBMnzBMLGRayFqBkx+WqU8nBhBWYVacZtm8Fg4ldwshnLc+K8zBb7K9MowbmcWQtMH8Z9/SFiMgyQlAtJeCyNhEi5/A/YmN8NZOwefr9VuP7aC0LiQV1mJMjw5jAmvBGWN02HLY+qd6EVeJoKyOfHmec83vH/fy3WIaSuxSMjooA59VZ9hMgmKsj+P41EJtmaVz0tT/JtreiWhWJ5SWYv6x+M7p7D4UXdyYdCEougRxlAWGbyUVz7Eaq+4FCdyzb5egAbMAAL2zEftt1AJD2topcNyRzOL0c7OGN75V2PfM4t5xSb+YKmF5qeypSgks5OvHCH76mWh7g0yHAg7skXMZcov/OYM+8lnue+y7STOkp1cTAcaGJM/MOWSE= ceph-9150213e-023a-11ef-bc93-c7b262605968 2024-04-24T13:01:15.100 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:01:14 smithi069 ceph-mon[21800]: from='client.14134 -' entity='client.admin' cmd=[{"prefix": "cephadm generate-key", "target": ["mon-mgr", ""]}]: dispatch 2024-04-24T13:01:15.100 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:01:14 smithi069 ceph-mon[21800]: Generating ssh key... 2024-04-24T13:01:15.300 INFO:teuthology.orchestra.run.smithi069.stderr:Wrote public SSH key to /home/ubuntu/cephtest/ceph.pub 2024-04-24T13:01:15.301 INFO:teuthology.orchestra.run.smithi069.stderr:Adding key to root@localhost authorized_keys... 2024-04-24T13:01:15.301 INFO:teuthology.orchestra.run.smithi069.stderr:Adding host smithi069... 2024-04-24T13:01:15.964 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:01:15 smithi069 ceph-mon[21800]: from='client.14136 -' entity='client.admin' cmd=[{"prefix": "cephadm get-pub-key", "target": ["mon-mgr", ""]}]: dispatch 2024-04-24T13:01:16.729 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: Added host 'smithi069' with addr '172.21.15.69' 2024-04-24T13:01:17.038 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:01:16 smithi069 ceph-mon[21800]: from='client.14138 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi069", "addr": "172.21.15.69", "target": ["mon-mgr", ""]}]: dispatch 2024-04-24T13:01:17.038 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:01:16 smithi069 ceph-mon[21800]: from='mgr.14118 172.21.15.69:0/4119475000' entity='mgr.smithi069.vspncx' 2024-04-24T13:01:17.038 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:01:16 smithi069 ceph-mon[21800]: from='mgr.14118 172.21.15.69:0/4119475000' entity='mgr.smithi069.vspncx' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-24T13:01:17.340 INFO:teuthology.orchestra.run.smithi069.stderr:Deploying mon service with default placement... 2024-04-24T13:01:17.972 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:01:17 smithi069 ceph-mon[21800]: Deploying cephadm binary to smithi069 2024-04-24T13:01:17.972 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:01:17 smithi069 ceph-mon[21800]: Added host smithi069 2024-04-24T13:01:17.972 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: Scheduled mon update... 2024-04-24T13:01:18.573 INFO:teuthology.orchestra.run.smithi069.stderr:Deploying mgr service with default placement... 2024-04-24T13:01:19.270 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:01:18 smithi069 ceph-mon[21800]: from='client.14140 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "target": ["mon-mgr", ""]}]: dispatch 2024-04-24T13:01:19.270 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:01:18 smithi069 ceph-mon[21800]: Saving service mon spec with placement count:5 2024-04-24T13:01:19.270 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:01:18 smithi069 ceph-mon[21800]: from='mgr.14118 172.21.15.69:0/4119475000' entity='mgr.smithi069.vspncx' 2024-04-24T13:01:19.378 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: Scheduled mgr update... 2024-04-24T13:01:20.199 INFO:teuthology.orchestra.run.smithi069.stderr:Deploying crash service with default placement... 2024-04-24T13:01:20.507 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:01:20 smithi069 ceph-mon[21800]: from='client.14142 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "target": ["mon-mgr", ""]}]: dispatch 2024-04-24T13:01:20.507 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:01:20 smithi069 ceph-mon[21800]: Saving service mgr spec with placement count:2 2024-04-24T13:01:20.508 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:01:20 smithi069 ceph-mon[21800]: from='mgr.14118 172.21.15.69:0/4119475000' entity='mgr.smithi069.vspncx' 2024-04-24T13:01:20.796 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: Scheduled crash update... 2024-04-24T13:01:21.440 INFO:teuthology.orchestra.run.smithi069.stderr:Deploying prometheus service with default placement... 2024-04-24T13:01:22.060 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:01:21 smithi069 ceph-mon[21800]: from='client.14144 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "crash", "target": ["mon-mgr", ""]}]: dispatch 2024-04-24T13:01:22.060 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:01:21 smithi069 ceph-mon[21800]: Saving service crash spec with placement * 2024-04-24T13:01:22.060 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:01:21 smithi069 ceph-mon[21800]: from='mgr.14118 172.21.15.69:0/4119475000' entity='mgr.smithi069.vspncx' 2024-04-24T13:01:22.060 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: Scheduled prometheus update... 2024-04-24T13:01:22.843 INFO:teuthology.orchestra.run.smithi069.stderr:Deploying grafana service with default placement... 2024-04-24T13:01:23.100 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:01:23 smithi069 ceph-mon[21800]: from='client.14146 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "target": ["mon-mgr", ""]}]: dispatch 2024-04-24T13:01:23.100 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:01:23 smithi069 ceph-mon[21800]: Saving service prometheus spec with placement count:1 2024-04-24T13:01:23.100 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:01:23 smithi069 ceph-mon[21800]: from='mgr.14118 172.21.15.69:0/4119475000' entity='mgr.smithi069.vspncx' 2024-04-24T13:01:23.100 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:01:23 smithi069 ceph-mon[21800]: from='mgr.14118 172.21.15.69:0/4119475000' entity='mgr.smithi069.vspncx' 2024-04-24T13:01:23.559 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: Scheduled grafana update... 2024-04-24T13:01:24.316 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:01:24 smithi069 ceph-mon[21800]: from='mgr.14118 172.21.15.69:0/4119475000' entity='mgr.smithi069.vspncx' 2024-04-24T13:01:24.316 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:01:24 smithi069 ceph-mon[21800]: from='client.14148 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "target": ["mon-mgr", ""]}]: dispatch 2024-04-24T13:01:24.316 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:01:24 smithi069 ceph-mon[21800]: Saving service grafana spec with placement count:1 2024-04-24T13:01:24.316 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:01:24 smithi069 ceph-mon[21800]: from='mgr.14118 172.21.15.69:0/4119475000' entity='mgr.smithi069.vspncx' 2024-04-24T13:01:24.678 INFO:teuthology.orchestra.run.smithi069.stderr:Deploying node-exporter service with default placement... 2024-04-24T13:01:25.328 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:01:25 smithi069 ceph-mon[21800]: from='mgr.14118 172.21.15.69:0/4119475000' entity='mgr.smithi069.vspncx' 2024-04-24T13:01:25.514 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:01:25 smithi069 ceph-mon[21800]: from='mgr.14118 172.21.15.69:0/4119475000' entity='mgr.smithi069.vspncx' 2024-04-24T13:01:25.515 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: Scheduled node-exporter update... 2024-04-24T13:01:26.339 INFO:teuthology.orchestra.run.smithi069.stderr:Deploying alertmanager service with default placement... 2024-04-24T13:01:26.600 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:01:26 smithi069 ceph-mon[21800]: from='client.14150 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "node-exporter", "target": ["mon-mgr", ""]}]: dispatch 2024-04-24T13:01:26.601 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:01:26 smithi069 ceph-mon[21800]: Saving service node-exporter spec with placement * 2024-04-24T13:01:26.601 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:01:26 smithi069 ceph-mon[21800]: from='mgr.14118 172.21.15.69:0/4119475000' entity='mgr.smithi069.vspncx' 2024-04-24T13:01:26.957 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: Scheduled alertmanager update... 2024-04-24T13:01:28.114 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:01:27 smithi069 ceph-mon[21800]: from='client.14152 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "target": ["mon-mgr", ""]}]: dispatch 2024-04-24T13:01:28.115 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:01:27 smithi069 ceph-mon[21800]: Saving service alertmanager spec with placement count:1 2024-04-24T13:01:28.115 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:01:27 smithi069 ceph-mon[21800]: from='mgr.14118 172.21.15.69:0/4119475000' entity='mgr.smithi069.vspncx' 2024-04-24T13:01:29.596 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:01:29 smithi069 ceph-mon[21800]: from='client.? 172.21.15.69:0/3054717400' entity='client.admin' 2024-04-24T13:01:29.596 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:01:29 smithi069 ceph-mon[21800]: pgmap v3: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-24T13:01:30.627 INFO:teuthology.orchestra.run.smithi069.stderr:Enabling the dashboard module... 2024-04-24T13:01:30.876 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:01:30 smithi069 ceph-mon[21800]: from='client.? 172.21.15.69:0/3198594861' entity='client.admin' 2024-04-24T13:01:32.220 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:01:31 smithi069 ceph-mon[21800]: pgmap v4: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-24T13:01:32.220 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:01:31 smithi069 ceph-mon[21800]: from='client.? 172.21.15.69:0/812346057' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "dashboard"}]: dispatch 2024-04-24T13:01:33.350 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:01:33 smithi069 ceph-mon[21800]: from='client.? 172.21.15.69:0/812346057' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "dashboard"}]': finished 2024-04-24T13:01:33.350 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:01:33 smithi069 ceph-mon[21800]: mgrmap e9: smithi069.vspncx(active, since 23s) 2024-04-24T13:01:33.513 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: { 2024-04-24T13:01:33.513 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "epoch": 9, 2024-04-24T13:01:33.514 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "available": true, 2024-04-24T13:01:33.514 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "active_name": "smithi069.vspncx", 2024-04-24T13:01:33.514 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "num_standby": 0 2024-04-24T13:01:33.514 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: } 2024-04-24T13:01:34.146 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:01:34 smithi069 ceph-mon[21800]: from='client.? 172.21.15.69:0/2309913354' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2024-04-24T13:01:34.200 INFO:teuthology.orchestra.run.smithi069.stderr:Waiting for the mgr to restart... 2024-04-24T13:01:34.200 INFO:teuthology.orchestra.run.smithi069.stderr:Waiting for mgr epoch 9... 2024-04-24T13:01:38.600 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:01:38 smithi069 ceph-mon[21800]: Active manager daemon smithi069.vspncx restarted 2024-04-24T13:01:38.600 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:01:38 smithi069 ceph-mon[21800]: Activating manager daemon smithi069.vspncx 2024-04-24T13:01:38.600 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:01:38 smithi069 ceph-mon[21800]: osdmap e3: 0 total, 0 up, 0 in 2024-04-24T13:01:39.178 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: { 2024-04-24T13:01:39.179 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "mgrmap_epoch": 11, 2024-04-24T13:01:39.179 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: "initialized": true 2024-04-24T13:01:39.179 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: } 2024-04-24T13:01:39.350 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:01:39 smithi069 ceph-mon[21800]: mgrmap e10: smithi069.vspncx(active, starting, since 0.0552973s) 2024-04-24T13:01:39.351 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:01:39 smithi069 ceph-mon[21800]: from='mgr.14160 172.21.15.69:0/2146052984' entity='mgr.smithi069.vspncx' cmd=[{"prefix": "mon metadata", "id": "smithi069"}]: dispatch 2024-04-24T13:01:39.351 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:01:39 smithi069 ceph-mon[21800]: from='mgr.14160 172.21.15.69:0/2146052984' entity='mgr.smithi069.vspncx' cmd=[{"prefix": "mgr metadata", "who": "smithi069.vspncx", "id": "smithi069.vspncx"}]: dispatch 2024-04-24T13:01:39.351 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:01:39 smithi069 ceph-mon[21800]: from='mgr.14160 172.21.15.69:0/2146052984' entity='mgr.smithi069.vspncx' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-24T13:01:39.351 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:01:39 smithi069 ceph-mon[21800]: from='mgr.14160 172.21.15.69:0/2146052984' entity='mgr.smithi069.vspncx' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-24T13:01:39.351 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:01:39 smithi069 ceph-mon[21800]: from='mgr.14160 172.21.15.69:0/2146052984' entity='mgr.smithi069.vspncx' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-24T13:01:39.351 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:01:39 smithi069 ceph-mon[21800]: Manager daemon smithi069.vspncx is now available 2024-04-24T13:01:39.351 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:01:39 smithi069 ceph-mon[21800]: from='mgr.14160 172.21.15.69:0/2146052984' entity='mgr.smithi069.vspncx' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-24T13:01:39.351 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:01:39 smithi069 ceph-mon[21800]: from='mgr.14160 172.21.15.69:0/2146052984' entity='mgr.smithi069.vspncx' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-24T13:01:39.351 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:01:39 smithi069 ceph-mon[21800]: from='mgr.14160 172.21.15.69:0/2146052984' entity='mgr.smithi069.vspncx' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi069.vspncx/mirror_snapshot_schedule"}]: dispatch 2024-04-24T13:01:39.351 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:01:39 smithi069 ceph-mon[21800]: from='mgr.14160 172.21.15.69:0/2146052984' entity='mgr.smithi069.vspncx' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi069.vspncx/trash_purge_schedule"}]: dispatch 2024-04-24T13:01:39.351 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:01:39 smithi069 ceph-mon[21800]: [24/Apr/2024:13:01:38] ENGINE Bus STARTING 2024-04-24T13:01:39.351 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:01:39 smithi069 ceph-mon[21800]: [24/Apr/2024:13:01:38] ENGINE Serving on https://172.21.15.69:7150 2024-04-24T13:01:39.351 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:01:39 smithi069 ceph-mon[21800]: [24/Apr/2024:13:01:38] ENGINE Bus STARTED 2024-04-24T13:01:39.352 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:01:39 smithi069 ceph-mon[21800]: from='mgr.14160 172.21.15.69:0/2146052984' entity='mgr.smithi069.vspncx' 2024-04-24T13:01:39.935 INFO:teuthology.orchestra.run.smithi069.stderr:mgr epoch 9 is available 2024-04-24T13:01:39.936 INFO:teuthology.orchestra.run.smithi069.stderr:Generating a dashboard self-signed certificate... 2024-04-24T13:01:40.286 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:01:40 smithi069 ceph-mon[21800]: from='client.14164 -' entity='client.admin' cmd=[{"prefix": "get_command_descriptions"}]: dispatch 2024-04-24T13:01:40.286 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:01:40 smithi069 ceph-mon[21800]: mgrmap e11: smithi069.vspncx(active, since 1.06296s) 2024-04-24T13:01:40.286 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:01:40 smithi069 ceph-mon[21800]: from='client.14164 -' entity='client.admin' cmd=[{"prefix": "mgr_status"}]: dispatch 2024-04-24T13:01:40.286 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:01:40 smithi069 ceph-mon[21800]: from='mgr.14160 172.21.15.69:0/2146052984' entity='mgr.smithi069.vspncx' 2024-04-24T13:01:40.633 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: Self-signed certificate created 2024-04-24T13:01:41.251 INFO:teuthology.orchestra.run.smithi069.stderr:Creating initial admin user... 2024-04-24T13:01:41.836 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:01:41 smithi069 ceph-mon[21800]: from='client.14172 -' entity='client.admin' cmd=[{"prefix": "dashboard create-self-signed-cert", "target": ["mon-mgr", ""]}]: dispatch 2024-04-24T13:01:41.836 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:01:41 smithi069 ceph-mon[21800]: from='mgr.14160 172.21.15.69:0/2146052984' entity='mgr.smithi069.vspncx' 2024-04-24T13:01:41.836 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:01:41 smithi069 ceph-mon[21800]: from='mgr.14160 172.21.15.69:0/2146052984' entity='mgr.smithi069.vspncx' 2024-04-24T13:01:41.836 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:01:41 smithi069 ceph-mon[21800]: from='mgr.14160 172.21.15.69:0/2146052984' entity='mgr.smithi069.vspncx' 2024-04-24T13:01:41.836 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:01:41 smithi069 ceph-mon[21800]: mgrmap e12: smithi069.vspncx(active, since 2s) 2024-04-24T13:01:42.084 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: {"username": "admin", "password": "$2b$12$H0beNIcpU3KVmFgrfW9Ote4fFA6mBPjOGnRAA/mjoPjwhGI1QwUvG", "roles": ["administrator"], "name": null, "email": null, "lastUpdate": 1713963702, "enabled": true, "pwdExpirationDate": null, "pwdUpdateRequired": true} 2024-04-24T13:01:43.036 INFO:teuthology.orchestra.run.smithi069.stderr:Fetching dashboard port number... 2024-04-24T13:01:43.350 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:01:43 smithi069 ceph-mon[21800]: from='client.14174 -' entity='client.admin' cmd=[{"prefix": "dashboard ac-user-create", "username": "admin", "rolename": "administrator", "force_password": true, "pwd_update_required": true, "target": ["mon-mgr", ""]}]: dispatch 2024-04-24T13:01:43.350 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:01:43 smithi069 ceph-mon[21800]: from='mgr.14160 172.21.15.69:0/2146052984' entity='mgr.smithi069.vspncx' 2024-04-24T13:01:44.192 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: 8443 2024-04-24T13:01:44.812 INFO:teuthology.orchestra.run.smithi069.stderr:systemctl: disabled 2024-04-24T13:01:44.819 INFO:teuthology.orchestra.run.smithi069.stderr:systemctl: inactive 2024-04-24T13:01:44.827 INFO:teuthology.orchestra.run.smithi069.stderr:firewalld.service is not enabled 2024-04-24T13:01:44.827 INFO:teuthology.orchestra.run.smithi069.stderr:Not possible to open ports <[8443]>. firewalld.service is not available 2024-04-24T13:01:44.828 INFO:teuthology.orchestra.run.smithi069.stderr:Ceph Dashboard is now available at: 2024-04-24T13:01:44.828 INFO:teuthology.orchestra.run.smithi069.stderr: 2024-04-24T13:01:44.828 INFO:teuthology.orchestra.run.smithi069.stderr: URL: https://smithi069:8443/ 2024-04-24T13:01:44.829 INFO:teuthology.orchestra.run.smithi069.stderr: User: admin 2024-04-24T13:01:44.829 INFO:teuthology.orchestra.run.smithi069.stderr: Password: 7yraow9dnz 2024-04-24T13:01:44.829 INFO:teuthology.orchestra.run.smithi069.stderr: 2024-04-24T13:01:44.829 INFO:teuthology.orchestra.run.smithi069.stderr:Enabling autotune for osd_memory_target 2024-04-24T13:01:45.354 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:01:45 smithi069 ceph-mon[21800]: mgrmap e13: smithi069.vspncx(active, since 6s) 2024-04-24T13:01:45.355 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:01:45 smithi069 ceph-mon[21800]: from='client.? 172.21.15.69:0/3833881868' entity='client.admin' cmd=[{"prefix": "config get", "who": "mgr", "key": "mgr/dashboard/ssl_server_port"}]: dispatch 2024-04-24T13:01:46.837 INFO:teuthology.orchestra.run.smithi069.stderr:/usr/bin/ceph: set mgr/dashboard/cluster/status 2024-04-24T13:01:47.527 INFO:teuthology.orchestra.run.smithi069.stderr:You can access the Ceph CLI with: 2024-04-24T13:01:47.528 INFO:teuthology.orchestra.run.smithi069.stderr: 2024-04-24T13:01:47.528 INFO:teuthology.orchestra.run.smithi069.stderr: sudo /home/ubuntu/cephtest/cephadm shell --fsid 9150213e-023a-11ef-bc93-c7b262605968 -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring 2024-04-24T13:01:47.528 INFO:teuthology.orchestra.run.smithi069.stderr: 2024-04-24T13:01:47.528 INFO:teuthology.orchestra.run.smithi069.stderr:Please consider enabling telemetry to help improve Ceph: 2024-04-24T13:01:47.528 INFO:teuthology.orchestra.run.smithi069.stderr: 2024-04-24T13:01:47.528 INFO:teuthology.orchestra.run.smithi069.stderr: ceph telemetry on 2024-04-24T13:01:47.528 INFO:teuthology.orchestra.run.smithi069.stderr: 2024-04-24T13:01:47.528 INFO:teuthology.orchestra.run.smithi069.stderr:For more information see: 2024-04-24T13:01:47.528 INFO:teuthology.orchestra.run.smithi069.stderr: 2024-04-24T13:01:47.528 INFO:teuthology.orchestra.run.smithi069.stderr: https://docs.ceph.com/docs/master/mgr/telemetry/ 2024-04-24T13:01:47.528 INFO:teuthology.orchestra.run.smithi069.stderr: 2024-04-24T13:01:47.528 INFO:teuthology.orchestra.run.smithi069.stderr:Bootstrap complete. 2024-04-24T13:01:47.559 INFO:tasks.cephadm:Fetching config... 2024-04-24T13:01:47.560 DEBUG:teuthology.orchestra.run.smithi069:> set -ex 2024-04-24T13:01:47.560 DEBUG:teuthology.orchestra.run.smithi069:> dd if=/etc/ceph/ceph.conf of=/dev/stdout 2024-04-24T13:01:47.575 INFO:tasks.cephadm:Fetching client.admin keyring... 2024-04-24T13:01:47.575 DEBUG:teuthology.orchestra.run.smithi069:> set -ex 2024-04-24T13:01:47.575 DEBUG:teuthology.orchestra.run.smithi069:> dd if=/etc/ceph/ceph.client.admin.keyring of=/dev/stdout 2024-04-24T13:01:47.631 INFO:tasks.cephadm:Fetching mon keyring... 2024-04-24T13:01:47.631 DEBUG:teuthology.orchestra.run.smithi069:> set -ex 2024-04-24T13:01:47.631 DEBUG:teuthology.orchestra.run.smithi069:> sudo dd if=/var/lib/ceph/9150213e-023a-11ef-bc93-c7b262605968/mon.smithi069/keyring of=/dev/stdout 2024-04-24T13:01:47.694 INFO:tasks.cephadm:Fetching pub ssh key... 2024-04-24T13:01:47.694 DEBUG:teuthology.orchestra.run.smithi069:> set -ex 2024-04-24T13:01:47.694 DEBUG:teuthology.orchestra.run.smithi069:> dd if=/home/ubuntu/cephtest/ceph.pub of=/dev/stdout 2024-04-24T13:01:47.751 INFO:tasks.cephadm:Installing pub ssh key for root users... 2024-04-24T13:01:47.752 DEBUG:teuthology.orchestra.run.smithi069:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDT1MvGN6Xh22oyDaKzv/wFONtnFo7QGIwoRv/R3GhL/LbR7sNIvtPOygfwyXzIVi1oaeZVPkCxeu+UgGAqPUIqi+l9wDhDaVNhdaW2xhsUpqFFsLu5Ed6dgZ7QZK8itZT6aptwsBMnzBMLGRayFqBkx+WqU8nBhBWYVacZtm8Fg4ldwshnLc+K8zBb7K9MowbmcWQtMH8Z9/SFiMgyQlAtJeCyNhEi5/A/YmN8NZOwefr9VuP7aC0LiQV1mJMjw5jAmvBGWN02HLY+qd6EVeJoKyOfHmec83vH/fy3WIaSuxSMjooA59VZ9hMgmKsj+P41EJtmaVz0tT/JtreiWhWJ5SWYv6x+M7p7D4UXdyYdCEougRxlAWGbyUVz7Eaq+4FCdyzb5egAbMAAL2zEftt1AJD2topcNyRzOL0c7OGN75V2PfM4t5xSb+YKmF5qeypSgks5OvHCH76mWh7g0yHAg7skXMZcov/OYM+8lnue+y7STOkp1cTAcaGJM/MOWSE= ceph-9150213e-023a-11ef-bc93-c7b262605968' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2024-04-24T13:01:47.823 INFO:teuthology.orchestra.run.smithi069.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDT1MvGN6Xh22oyDaKzv/wFONtnFo7QGIwoRv/R3GhL/LbR7sNIvtPOygfwyXzIVi1oaeZVPkCxeu+UgGAqPUIqi+l9wDhDaVNhdaW2xhsUpqFFsLu5Ed6dgZ7QZK8itZT6aptwsBMnzBMLGRayFqBkx+WqU8nBhBWYVacZtm8Fg4ldwshnLc+K8zBb7K9MowbmcWQtMH8Z9/SFiMgyQlAtJeCyNhEi5/A/YmN8NZOwefr9VuP7aC0LiQV1mJMjw5jAmvBGWN02HLY+qd6EVeJoKyOfHmec83vH/fy3WIaSuxSMjooA59VZ9hMgmKsj+P41EJtmaVz0tT/JtreiWhWJ5SWYv6x+M7p7D4UXdyYdCEougRxlAWGbyUVz7Eaq+4FCdyzb5egAbMAAL2zEftt1AJD2topcNyRzOL0c7OGN75V2PfM4t5xSb+YKmF5qeypSgks5OvHCH76mWh7g0yHAg7skXMZcov/OYM+8lnue+y7STOkp1cTAcaGJM/MOWSE= ceph-9150213e-023a-11ef-bc93-c7b262605968 2024-04-24T13:01:47.833 DEBUG:teuthology.orchestra.run.smithi114:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDT1MvGN6Xh22oyDaKzv/wFONtnFo7QGIwoRv/R3GhL/LbR7sNIvtPOygfwyXzIVi1oaeZVPkCxeu+UgGAqPUIqi+l9wDhDaVNhdaW2xhsUpqFFsLu5Ed6dgZ7QZK8itZT6aptwsBMnzBMLGRayFqBkx+WqU8nBhBWYVacZtm8Fg4ldwshnLc+K8zBb7K9MowbmcWQtMH8Z9/SFiMgyQlAtJeCyNhEi5/A/YmN8NZOwefr9VuP7aC0LiQV1mJMjw5jAmvBGWN02HLY+qd6EVeJoKyOfHmec83vH/fy3WIaSuxSMjooA59VZ9hMgmKsj+P41EJtmaVz0tT/JtreiWhWJ5SWYv6x+M7p7D4UXdyYdCEougRxlAWGbyUVz7Eaq+4FCdyzb5egAbMAAL2zEftt1AJD2topcNyRzOL0c7OGN75V2PfM4t5xSb+YKmF5qeypSgks5OvHCH76mWh7g0yHAg7skXMZcov/OYM+8lnue+y7STOkp1cTAcaGJM/MOWSE= ceph-9150213e-023a-11ef-bc93-c7b262605968' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2024-04-24T13:01:47.874 INFO:teuthology.orchestra.run.smithi114.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDT1MvGN6Xh22oyDaKzv/wFONtnFo7QGIwoRv/R3GhL/LbR7sNIvtPOygfwyXzIVi1oaeZVPkCxeu+UgGAqPUIqi+l9wDhDaVNhdaW2xhsUpqFFsLu5Ed6dgZ7QZK8itZT6aptwsBMnzBMLGRayFqBkx+WqU8nBhBWYVacZtm8Fg4ldwshnLc+K8zBb7K9MowbmcWQtMH8Z9/SFiMgyQlAtJeCyNhEi5/A/YmN8NZOwefr9VuP7aC0LiQV1mJMjw5jAmvBGWN02HLY+qd6EVeJoKyOfHmec83vH/fy3WIaSuxSMjooA59VZ9hMgmKsj+P41EJtmaVz0tT/JtreiWhWJ5SWYv6x+M7p7D4UXdyYdCEougRxlAWGbyUVz7Eaq+4FCdyzb5egAbMAAL2zEftt1AJD2topcNyRzOL0c7OGN75V2PfM4t5xSb+YKmF5qeypSgks5OvHCH76mWh7g0yHAg7skXMZcov/OYM+8lnue+y7STOkp1cTAcaGJM/MOWSE= ceph-9150213e-023a-11ef-bc93-c7b262605968 2024-04-24T13:01:47.886 DEBUG:teuthology.orchestra.run.smithi069:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9150213e-023a-11ef-bc93-c7b262605968 -- ceph config set mgr mgr/cephadm/allow_ptrace true 2024-04-24T13:01:48.062 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:01:47 smithi069 ceph-mon[21800]: from='client.? 172.21.15.69:0/4229399246' entity='client.admin' 2024-04-24T13:01:51.100 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:01:50 smithi069 ceph-mon[21800]: from='mgr.14160 172.21.15.69:0/2146052984' entity='mgr.smithi069.vspncx' 2024-04-24T13:01:51.101 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:01:50 smithi069 ceph-mon[21800]: from='mgr.14160 172.21.15.69:0/2146052984' entity='mgr.smithi069.vspncx' cmd=[{"prefix": "config rm", "who": "osd/host:smithi069", "name": "osd_memory_target"}]: dispatch 2024-04-24T13:01:51.101 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:01:50 smithi069 ceph-mon[21800]: from='mgr.14160 172.21.15.69:0/2146052984' entity='mgr.smithi069.vspncx' 2024-04-24T13:01:51.101 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:01:50 smithi069 ceph-mon[21800]: Deploying daemon alertmanager.smithi069 on smithi069 2024-04-24T13:01:51.101 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:01:50 smithi069 ceph-mon[21800]: from='client.? 172.21.15.69:0/711968838' entity='client.admin' 2024-04-24T13:01:51.498 INFO:tasks.cephadm:Distributing conf and client.admin keyring to all hosts + 0755 2024-04-24T13:01:51.498 DEBUG:teuthology.orchestra.run.smithi069:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9150213e-023a-11ef-bc93-c7b262605968 -- ceph orch client-keyring set client.admin '*' --mode 0755 2024-04-24T13:01:55.099 INFO:tasks.cephadm:Writing (initial) conf and keyring to smithi114 2024-04-24T13:01:55.099 DEBUG:teuthology.orchestra.run.smithi114:> set -ex 2024-04-24T13:01:55.099 DEBUG:teuthology.orchestra.run.smithi114:> dd of=/etc/ceph/ceph.conf 2024-04-24T13:01:55.117 DEBUG:teuthology.orchestra.run.smithi114:> set -ex 2024-04-24T13:01:55.118 DEBUG:teuthology.orchestra.run.smithi114:> dd of=/etc/ceph/ceph.client.admin.keyring 2024-04-24T13:01:55.174 INFO:tasks.cephadm:Adding host smithi114 to orchestrator... 2024-04-24T13:01:55.174 DEBUG:teuthology.orchestra.run.smithi069:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9150213e-023a-11ef-bc93-c7b262605968 -- ceph orch host add smithi114 2024-04-24T13:01:55.350 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:01:55 smithi069 ceph-mon[21800]: from='client.14184 -' entity='client.admin' cmd=[{"prefix": "orch client-keyring set", "entity": "client.admin", "placement": "*", "mode": "0755", "target": ["mon-mgr", ""]}]: dispatch 2024-04-24T13:01:55.350 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:01:55 smithi069 ceph-mon[21800]: from='mgr.14160 172.21.15.69:0/2146052984' entity='mgr.smithi069.vspncx' 2024-04-24T13:01:59.350 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:01:59 smithi069 ceph-mon[21800]: pgmap v3: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-24T13:02:01.948 INFO:teuthology.orchestra.run.smithi069.stdout:Added host 'smithi114' with addr '172.21.15.114' 2024-04-24T13:02:02.306 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:02:02 smithi069 ceph-mon[21800]: pgmap v4: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-24T13:02:02.306 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:02:02 smithi069 ceph-mon[21800]: from='client.14186 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi114", "target": ["mon-mgr", ""]}]: dispatch 2024-04-24T13:02:02.306 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:02:02 smithi069 ceph-mon[21800]: from='mgr.14160 172.21.15.69:0/2146052984' entity='mgr.smithi069.vspncx' 2024-04-24T13:02:02.306 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:02:02 smithi069 ceph-mon[21800]: from='mgr.14160 172.21.15.69:0/2146052984' entity='mgr.smithi069.vspncx' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi069", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-04-24T13:02:02.307 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:02:02 smithi069 ceph-mon[21800]: from='mgr.14160 172.21.15.69:0/2146052984' entity='mgr.smithi069.vspncx' cmd='[{"prefix": "auth get-or-create", "entity": "client.crash.smithi069", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]': finished 2024-04-24T13:02:02.307 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:02:02 smithi069 ceph-mon[21800]: from='mgr.14160 172.21.15.69:0/2146052984' entity='mgr.smithi069.vspncx' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-24T13:02:02.307 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:02:02 smithi069 ceph-mon[21800]: from='mgr.14160 172.21.15.69:0/2146052984' entity='mgr.smithi069.vspncx' 2024-04-24T13:02:02.600 DEBUG:teuthology.orchestra.run.smithi069:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9150213e-023a-11ef-bc93-c7b262605968 -- ceph orch host ls --format=json 2024-04-24T13:02:03.350 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:02:03 smithi069 ceph-mon[21800]: Deploying cephadm binary to smithi114 2024-04-24T13:02:03.351 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:02:03 smithi069 ceph-mon[21800]: Deploying daemon crash.smithi069 on smithi069 2024-04-24T13:02:03.351 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:02:03 smithi069 ceph-mon[21800]: Added host smithi114 2024-04-24T13:02:04.150 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:02:04 smithi069 ceph-mon[21800]: pgmap v5: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-24T13:02:04.151 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:02:04 smithi069 ceph-mon[21800]: from='mgr.14160 172.21.15.69:0/2146052984' entity='mgr.smithi069.vspncx' 2024-04-24T13:02:05.463 INFO:teuthology.orchestra.run.smithi069.stdout: 2024-04-24T13:02:05.463 INFO:teuthology.orchestra.run.smithi069.stdout:[{"addr": "172.21.15.69", "hostname": "smithi069", "labels": [], "status": ""}, {"addr": "172.21.15.114", "hostname": "smithi114", "labels": [], "status": ""}] 2024-04-24T13:02:06.254 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:02:06 smithi069 ceph-mon[21800]: pgmap v6: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-24T13:02:06.569 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:02:06 smithi069 ceph-mon[21800]: from='mgr.14160 172.21.15.69:0/2146052984' entity='mgr.smithi069.vspncx' 2024-04-24T13:02:06.570 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:02:06 smithi069 ceph-mon[21800]: from='mgr.14160 172.21.15.69:0/2146052984' entity='mgr.smithi069.vspncx' 2024-04-24T13:02:06.570 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:02:06 smithi069 ceph-mon[21800]: from='mgr.14160 172.21.15.69:0/2146052984' entity='mgr.smithi069.vspncx' 2024-04-24T13:02:06.570 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:02:06 smithi069 ceph-mon[21800]: from='mgr.14160 172.21.15.69:0/2146052984' entity='mgr.smithi069.vspncx' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-04-24T13:02:06.570 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:02:06 smithi069 ceph-mon[21800]: from='mgr.14160 172.21.15.69:0/2146052984' entity='mgr.smithi069.vspncx' 2024-04-24T13:02:06.571 INFO:tasks.cephadm:Setting crush tunables to default 2024-04-24T13:02:06.572 DEBUG:teuthology.orchestra.run.smithi069:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9150213e-023a-11ef-bc93-c7b262605968 -- ceph osd crush tunables default 2024-04-24T13:02:07.190 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:02:07 smithi069 ceph-mon[21800]: from='client.14188 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-24T13:02:07.190 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:02:07 smithi069 ceph-mon[21800]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-04-24T13:02:07.190 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:02:07 smithi069 ceph-mon[21800]: Deploying daemon grafana.smithi069 on smithi069 2024-04-24T13:02:08.190 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:02:08 smithi069 ceph-mon[21800]: pgmap v7: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-24T13:02:09.229 INFO:teuthology.orchestra.run.smithi069.stderr:adjusted tunables profile to default 2024-04-24T13:02:09.554 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:02:09 smithi069 ceph-mon[21800]: pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-24T13:02:09.554 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:02:09 smithi069 ceph-mon[21800]: from='mgr.14160 172.21.15.69:0/2146052984' entity='mgr.smithi069.vspncx' 2024-04-24T13:02:09.554 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:02:09 smithi069 ceph-mon[21800]: from='client.? 172.21.15.69:0/546240074' entity='client.admin' cmd=[{"prefix": "osd crush tunables", "profile": "default"}]: dispatch 2024-04-24T13:02:09.911 INFO:tasks.cephadm:Adding mon.smithi069 on smithi069 2024-04-24T13:02:09.911 INFO:tasks.cephadm:Adding mon.smithi114 on smithi114 2024-04-24T13:02:09.911 DEBUG:teuthology.orchestra.run.smithi114:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9150213e-023a-11ef-bc93-c7b262605968 -- ceph orch apply mon '2;smithi069:172.21.15.69=smithi069;smithi114:172.21.15.114=smithi114' 2024-04-24T13:02:10.600 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:02:10 smithi069 ceph-mon[21800]: from='client.? 172.21.15.69:0/546240074' entity='client.admin' cmd='[{"prefix": "osd crush tunables", "profile": "default"}]': finished 2024-04-24T13:02:10.600 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:02:10 smithi069 ceph-mon[21800]: osdmap e4: 0 total, 0 up, 0 in 2024-04-24T13:02:11.664 INFO:teuthology.orchestra.run.smithi114.stdout:Scheduled mon update... 2024-04-24T13:02:12.160 DEBUG:teuthology.orchestra.run.smithi114:mon.smithi114> sudo journalctl -f -n 0 -u ceph-9150213e-023a-11ef-bc93-c7b262605968@mon.smithi114.service 2024-04-24T13:02:12.163 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-04-24T13:02:12.163 DEBUG:teuthology.orchestra.run.smithi114:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9150213e-023a-11ef-bc93-c7b262605968 -- ceph mon dump -f json 2024-04-24T13:02:12.350 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:02:12 smithi069 ceph-mon[21800]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-24T13:02:12.350 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:02:12 smithi069 ceph-mon[21800]: from='mgr.14160 172.21.15.69:0/2146052984' entity='mgr.smithi069.vspncx' 2024-04-24T13:02:13.350 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:02:13 smithi069 ceph-mon[21800]: from='client.14192 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "placement": "2;smithi069:172.21.15.69=smithi069;smithi114:172.21.15.114=smithi114", "target": ["mon-mgr", ""]}]: dispatch 2024-04-24T13:02:13.350 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:02:13 smithi069 ceph-mon[21800]: Saving service mon spec with placement smithi069:172.21.15.69=smithi069;smithi114:172.21.15.114=smithi114;count:2 2024-04-24T13:02:14.079 INFO:teuthology.orchestra.run.smithi114.stdout: 2024-04-24T13:02:14.079 INFO:teuthology.orchestra.run.smithi114.stdout:{"epoch":1,"fsid":"9150213e-023a-11ef-bc93-c7b262605968","modified":"2024-04-24T13:00:41.869875Z","created":"2024-04-24T13:00:41.869875Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi069","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.69:3300","nonce":0},{"type":"v1","addr":"172.21.15.69:6789","nonce":0}]},"addr":"172.21.15.69:6789/0","public_addr":"172.21.15.69:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-04-24T13:02:14.080 INFO:teuthology.orchestra.run.smithi114.stderr:dumped monmap epoch 1 2024-04-24T13:02:14.350 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:02:14 smithi069 ceph-mon[21800]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-24T13:02:15.350 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:02:15 smithi069 ceph-mon[21800]: from='client.? 172.21.15.114:0/4073857196' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-24T13:02:15.640 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-04-24T13:02:15.641 DEBUG:teuthology.orchestra.run.smithi114:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9150213e-023a-11ef-bc93-c7b262605968 -- ceph mon dump -f json 2024-04-24T13:02:16.351 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:02:16 smithi069 ceph-mon[21800]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-24T13:02:17.518 INFO:teuthology.orchestra.run.smithi114.stdout: 2024-04-24T13:02:17.518 INFO:teuthology.orchestra.run.smithi114.stdout:{"epoch":1,"fsid":"9150213e-023a-11ef-bc93-c7b262605968","modified":"2024-04-24T13:00:41.869875Z","created":"2024-04-24T13:00:41.869875Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi069","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.69:3300","nonce":0},{"type":"v1","addr":"172.21.15.69:6789","nonce":0}]},"addr":"172.21.15.69:6789/0","public_addr":"172.21.15.69:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-04-24T13:02:17.519 INFO:teuthology.orchestra.run.smithi114.stderr:dumped monmap epoch 1 2024-04-24T13:02:18.350 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:02:18 smithi069 ceph-mon[21800]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-24T13:02:18.351 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:02:18 smithi069 ceph-mon[21800]: from='client.? 172.21.15.114:0/3350503572' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-24T13:02:19.099 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-04-24T13:02:19.099 DEBUG:teuthology.orchestra.run.smithi114:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9150213e-023a-11ef-bc93-c7b262605968 -- ceph mon dump -f json 2024-04-24T13:02:20.237 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:02:20 smithi069 ceph-mon[21800]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-24T13:02:20.238 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:02:20 smithi069 ceph-mon[21800]: from='mgr.14160 172.21.15.69:0/2146052984' entity='mgr.smithi069.vspncx' 2024-04-24T13:02:20.980 INFO:teuthology.orchestra.run.smithi114.stdout: 2024-04-24T13:02:20.980 INFO:teuthology.orchestra.run.smithi114.stdout:{"epoch":1,"fsid":"9150213e-023a-11ef-bc93-c7b262605968","modified":"2024-04-24T13:00:41.869875Z","created":"2024-04-24T13:00:41.869875Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi069","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.69:3300","nonce":0},{"type":"v1","addr":"172.21.15.69:6789","nonce":0}]},"addr":"172.21.15.69:6789/0","public_addr":"172.21.15.69:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-04-24T13:02:20.981 INFO:teuthology.orchestra.run.smithi114.stderr:dumped monmap epoch 1 2024-04-24T13:02:21.350 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:02:21 smithi069 ceph-mon[21800]: Deploying daemon node-exporter.smithi069 on smithi069 2024-04-24T13:02:21.350 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:02:21 smithi069 ceph-mon[21800]: from='client.? 172.21.15.114:0/73992247' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-24T13:02:22.350 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:02:22 smithi069 ceph-mon[21800]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-24T13:02:22.638 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-04-24T13:02:22.639 DEBUG:teuthology.orchestra.run.smithi114:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9150213e-023a-11ef-bc93-c7b262605968 -- ceph mon dump -f json 2024-04-24T13:02:24.091 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:02:23 smithi069 ceph-mon[21800]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-24T13:02:24.091 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:02:23 smithi069 ceph-mon[21800]: from='mgr.14160 172.21.15.69:0/2146052984' entity='mgr.smithi069.vspncx' 2024-04-24T13:02:24.091 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:02:23 smithi069 ceph-mon[21800]: from='mgr.14160 172.21.15.69:0/2146052984' entity='mgr.smithi069.vspncx' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2024-04-24T13:02:24.091 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:02:23 smithi069 ceph-mon[21800]: from='mgr.14160 172.21.15.69:0/2146052984' entity='mgr.smithi069.vspncx' 2024-04-24T13:02:24.496 INFO:teuthology.orchestra.run.smithi114.stdout: 2024-04-24T13:02:24.496 INFO:teuthology.orchestra.run.smithi114.stdout:{"epoch":1,"fsid":"9150213e-023a-11ef-bc93-c7b262605968","modified":"2024-04-24T13:00:41.869875Z","created":"2024-04-24T13:00:41.869875Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi069","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.69:3300","nonce":0},{"type":"v1","addr":"172.21.15.69:6789","nonce":0}]},"addr":"172.21.15.69:6789/0","public_addr":"172.21.15.69:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-04-24T13:02:24.498 INFO:teuthology.orchestra.run.smithi114.stderr:dumped monmap epoch 1 2024-04-24T13:02:25.100 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:02:24 smithi069 ceph-mon[21800]: from='mgr.14160 172.21.15.69:0/2146052984' entity='mgr.smithi069.vspncx' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2024-04-24T13:02:25.100 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:02:24 smithi069 ceph-mon[21800]: mgrmap e14: smithi069.vspncx(active, since 45s) 2024-04-24T13:02:25.100 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:02:24 smithi069 ceph-mon[21800]: from='client.? 172.21.15.114:0/2568885727' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-24T13:02:26.091 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-04-24T13:02:26.091 DEBUG:teuthology.orchestra.run.smithi114:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9150213e-023a-11ef-bc93-c7b262605968 -- ceph mon dump -f json 2024-04-24T13:02:28.016 INFO:teuthology.orchestra.run.smithi114.stdout: 2024-04-24T13:02:28.016 INFO:teuthology.orchestra.run.smithi114.stdout:{"epoch":1,"fsid":"9150213e-023a-11ef-bc93-c7b262605968","modified":"2024-04-24T13:00:41.869875Z","created":"2024-04-24T13:00:41.869875Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi069","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.69:3300","nonce":0},{"type":"v1","addr":"172.21.15.69:6789","nonce":0}]},"addr":"172.21.15.69:6789/0","public_addr":"172.21.15.69:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-04-24T13:02:28.017 INFO:teuthology.orchestra.run.smithi114.stderr:dumped monmap epoch 1 2024-04-24T13:02:28.350 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:02:28 smithi069 ceph-mon[21800]: from='client.? 172.21.15.114:0/820551470' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-24T13:02:29.628 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-04-24T13:02:29.628 DEBUG:teuthology.orchestra.run.smithi114:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9150213e-023a-11ef-bc93-c7b262605968 -- ceph mon dump -f json 2024-04-24T13:02:30.850 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:02:30 smithi069 ceph-mon[21800]: Active manager daemon smithi069.vspncx restarted 2024-04-24T13:02:30.851 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:02:30 smithi069 ceph-mon[21800]: Activating manager daemon smithi069.vspncx 2024-04-24T13:02:30.851 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:02:30 smithi069 ceph-mon[21800]: osdmap e5: 0 total, 0 up, 0 in 2024-04-24T13:02:30.851 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:02:30 smithi069 ceph-mon[21800]: mgrmap e15: smithi069.vspncx(active, starting, since 0.0529045s) 2024-04-24T13:02:30.851 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:02:30 smithi069 ceph-mon[21800]: from='mgr.14200 172.21.15.69:0/2429389701' entity='mgr.smithi069.vspncx' cmd=[{"prefix": "mon metadata", "id": "smithi069"}]: dispatch 2024-04-24T13:02:30.851 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:02:30 smithi069 ceph-mon[21800]: from='mgr.14200 172.21.15.69:0/2429389701' entity='mgr.smithi069.vspncx' cmd=[{"prefix": "mgr metadata", "who": "smithi069.vspncx", "id": "smithi069.vspncx"}]: dispatch 2024-04-24T13:02:30.851 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:02:30 smithi069 ceph-mon[21800]: from='mgr.14200 172.21.15.69:0/2429389701' entity='mgr.smithi069.vspncx' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-24T13:02:30.851 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:02:30 smithi069 ceph-mon[21800]: from='mgr.14200 172.21.15.69:0/2429389701' entity='mgr.smithi069.vspncx' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-24T13:02:30.851 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:02:30 smithi069 ceph-mon[21800]: from='mgr.14200 172.21.15.69:0/2429389701' entity='mgr.smithi069.vspncx' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-24T13:02:30.851 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:02:30 smithi069 ceph-mon[21800]: Manager daemon smithi069.vspncx is now available 2024-04-24T13:02:30.851 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:02:30 smithi069 ceph-mon[21800]: from='mgr.14200 172.21.15.69:0/2429389701' entity='mgr.smithi069.vspncx' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-24T13:02:30.851 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:02:30 smithi069 ceph-mon[21800]: from='mgr.14200 172.21.15.69:0/2429389701' entity='mgr.smithi069.vspncx' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-24T13:02:30.851 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:02:30 smithi069 ceph-mon[21800]: from='mgr.14200 172.21.15.69:0/2429389701' entity='mgr.smithi069.vspncx' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-24T13:02:30.851 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:02:30 smithi069 ceph-mon[21800]: from='mgr.14200 172.21.15.69:0/2429389701' entity='mgr.smithi069.vspncx' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-24T13:02:30.852 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:02:30 smithi069 ceph-mon[21800]: from='mgr.14200 172.21.15.69:0/2429389701' entity='mgr.smithi069.vspncx' 2024-04-24T13:02:30.852 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:02:30 smithi069 ceph-mon[21800]: from='mgr.14200 172.21.15.69:0/2429389701' entity='mgr.smithi069.vspncx' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi069.vspncx/mirror_snapshot_schedule"}]: dispatch 2024-04-24T13:02:30.852 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:02:30 smithi069 ceph-mon[21800]: from='mgr.14200 172.21.15.69:0/2429389701' entity='mgr.smithi069.vspncx' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/smithi069.vspncx/trash_purge_schedule"}]: dispatch 2024-04-24T13:02:30.852 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:02:30 smithi069 ceph-mon[21800]: from='mgr.14200 172.21.15.69:0/2429389701' entity='mgr.smithi069.vspncx' 2024-04-24T13:02:30.852 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:02:30 smithi069 ceph-mon[21800]: from='mgr.14200 172.21.15.69:0/2429389701' entity='mgr.smithi069.vspncx' 2024-04-24T13:02:31.526 INFO:teuthology.orchestra.run.smithi114.stdout: 2024-04-24T13:02:31.526 INFO:teuthology.orchestra.run.smithi114.stdout:{"epoch":1,"fsid":"9150213e-023a-11ef-bc93-c7b262605968","modified":"2024-04-24T13:00:41.869875Z","created":"2024-04-24T13:00:41.869875Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi069","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.69:3300","nonce":0},{"type":"v1","addr":"172.21.15.69:6789","nonce":0}]},"addr":"172.21.15.69:6789/0","public_addr":"172.21.15.69:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-04-24T13:02:31.528 INFO:teuthology.orchestra.run.smithi114.stderr:dumped monmap epoch 1 2024-04-24T13:02:32.100 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:02:31 smithi069 ceph-mon[21800]: [24/Apr/2024:13:02:30] ENGINE Bus STARTING 2024-04-24T13:02:32.100 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:02:31 smithi069 ceph-mon[21800]: [24/Apr/2024:13:02:30] ENGINE Serving on https://172.21.15.69:7150 2024-04-24T13:02:32.100 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:02:31 smithi069 ceph-mon[21800]: [24/Apr/2024:13:02:30] ENGINE Bus STARTED 2024-04-24T13:02:32.100 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:02:31 smithi069 ceph-mon[21800]: mgrmap e16: smithi069.vspncx(active, since 1.05558s) 2024-04-24T13:02:32.100 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:02:31 smithi069 ceph-mon[21800]: from='mgr.14200 172.21.15.69:0/2429389701' entity='mgr.smithi069.vspncx' 2024-04-24T13:02:32.101 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:02:31 smithi069 ceph-mon[21800]: from='client.? 172.21.15.114:0/1758175602' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-24T13:02:33.189 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:02:33 smithi069 ceph-mon[21800]: mgrmap e17: smithi069.vspncx(active, since 2s) 2024-04-24T13:02:33.189 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:02:33 smithi069 ceph-mon[21800]: from='mgr.14200 172.21.15.69:0/2429389701' entity='mgr.smithi069.vspncx' 2024-04-24T13:02:33.195 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-04-24T13:02:33.195 DEBUG:teuthology.orchestra.run.smithi114:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9150213e-023a-11ef-bc93-c7b262605968 -- ceph mon dump -f json 2024-04-24T13:02:34.351 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:02:34 smithi069 ceph-mon[21800]: from='mgr.14200 172.21.15.69:0/2429389701' entity='mgr.smithi069.vspncx' 2024-04-24T13:02:34.351 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:02:34 smithi069 ceph-mon[21800]: from='mgr.14200 172.21.15.69:0/2429389701' entity='mgr.smithi069.vspncx' 2024-04-24T13:02:34.351 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:02:34 smithi069 ceph-mon[21800]: from='mgr.14200 172.21.15.69:0/2429389701' entity='mgr.smithi069.vspncx' cmd=[{"prefix": "config rm", "who": "osd/host:smithi069", "name": "osd_memory_target"}]: dispatch 2024-04-24T13:02:34.351 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:02:34 smithi069 ceph-mon[21800]: Updating smithi069:/etc/ceph/ceph.conf 2024-04-24T13:02:34.351 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:02:34 smithi069 ceph-mon[21800]: from='mgr.14200 172.21.15.69:0/2429389701' entity='mgr.smithi069.vspncx' 2024-04-24T13:02:35.496 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:02:35 smithi069 ceph-mon[21800]: Updating smithi069:/etc/ceph/ceph.client.admin.keyring 2024-04-24T13:02:35.496 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:02:35 smithi069 ceph-mon[21800]: from='mgr.14200 172.21.15.69:0/2429389701' entity='mgr.smithi069.vspncx' 2024-04-24T13:02:35.497 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:02:35 smithi069 ceph-mon[21800]: from='mgr.14200 172.21.15.69:0/2429389701' entity='mgr.smithi069.vspncx' 2024-04-24T13:02:35.497 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:02:35 smithi069 ceph-mon[21800]: from='mgr.14200 172.21.15.69:0/2429389701' entity='mgr.smithi069.vspncx' cmd=[{"prefix": "config rm", "who": "osd/host:smithi114", "name": "osd_memory_target"}]: dispatch 2024-04-24T13:02:35.497 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:02:35 smithi069 ceph-mon[21800]: from='mgr.14200 172.21.15.69:0/2429389701' entity='mgr.smithi069.vspncx' 2024-04-24T13:02:35.497 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:02:35 smithi069 ceph-mon[21800]: from='mgr.14200 172.21.15.69:0/2429389701' entity='mgr.smithi069.vspncx' cmd=[{"prefix": "auth get-or-create", "entity": "client.crash.smithi114", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]: dispatch 2024-04-24T13:02:35.497 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:02:35 smithi069 ceph-mon[21800]: from='mgr.14200 172.21.15.69:0/2429389701' entity='mgr.smithi069.vspncx' cmd='[{"prefix": "auth get-or-create", "entity": "client.crash.smithi114", "caps": ["mon", "profile crash", "mgr", "profile crash"]}]': finished 2024-04-24T13:02:35.497 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:02:35 smithi069 ceph-mon[21800]: from='mgr.14200 172.21.15.69:0/2429389701' entity='mgr.smithi069.vspncx' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-24T13:02:36.091 INFO:teuthology.orchestra.run.smithi114.stdout: 2024-04-24T13:02:36.091 INFO:teuthology.orchestra.run.smithi114.stdout:{"epoch":1,"fsid":"9150213e-023a-11ef-bc93-c7b262605968","modified":"2024-04-24T13:00:41.869875Z","created":"2024-04-24T13:00:41.869875Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi069","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.69:3300","nonce":0},{"type":"v1","addr":"172.21.15.69:6789","nonce":0}]},"addr":"172.21.15.69:6789/0","public_addr":"172.21.15.69:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-04-24T13:02:36.093 INFO:teuthology.orchestra.run.smithi114.stderr:dumped monmap epoch 1 2024-04-24T13:02:36.600 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:02:36 smithi069 ceph-mon[21800]: Deploying daemon crash.smithi114 on smithi114 2024-04-24T13:02:36.601 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:02:36 smithi069 ceph-mon[21800]: from='client.? 172.21.15.114:0/2287851652' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-24T13:02:38.569 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-04-24T13:02:38.570 DEBUG:teuthology.orchestra.run.smithi114:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9150213e-023a-11ef-bc93-c7b262605968 -- ceph mon dump -f json 2024-04-24T13:02:39.850 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:02:39 smithi069 ceph-mon[21800]: from='mgr.14200 172.21.15.69:0/2429389701' entity='mgr.smithi069.vspncx' 2024-04-24T13:02:39.851 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:02:39 smithi069 ceph-mon[21800]: from='mgr.14200 172.21.15.69:0/2429389701' entity='mgr.smithi069.vspncx' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.smithi114.oaurwi", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-24T13:02:39.851 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:02:39 smithi069 ceph-mon[21800]: from='mgr.14200 172.21.15.69:0/2429389701' entity='mgr.smithi069.vspncx' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.smithi114.oaurwi", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2024-04-24T13:02:39.851 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:02:39 smithi069 ceph-mon[21800]: from='mgr.14200 172.21.15.69:0/2429389701' entity='mgr.smithi069.vspncx' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-24T13:02:39.851 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:02:39 smithi069 ceph-mon[21800]: from='mgr.14200 172.21.15.69:0/2429389701' entity='mgr.smithi069.vspncx' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-24T13:02:39.851 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:02:39 smithi069 ceph-mon[21800]: Deploying daemon mgr.smithi114.oaurwi on smithi114 2024-04-24T13:02:41.100 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:02:40 smithi069 ceph-mon[21800]: from='mgr.14200 172.21.15.69:0/2429389701' entity='mgr.smithi069.vspncx' 2024-04-24T13:02:41.440 INFO:teuthology.orchestra.run.smithi114.stdout: 2024-04-24T13:02:41.440 INFO:teuthology.orchestra.run.smithi114.stdout:{"epoch":1,"fsid":"9150213e-023a-11ef-bc93-c7b262605968","modified":"2024-04-24T13:00:41.869875Z","created":"2024-04-24T13:00:41.869875Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi069","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.69:3300","nonce":0},{"type":"v1","addr":"172.21.15.69:6789","nonce":0}]},"addr":"172.21.15.69:6789/0","public_addr":"172.21.15.69:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-04-24T13:02:41.442 INFO:teuthology.orchestra.run.smithi114.stderr:dumped monmap epoch 1 2024-04-24T13:02:42.350 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:02:42 smithi069 ceph-mon[21800]: from='client.? 172.21.15.114:0/3017245832' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-24T13:02:42.350 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:02:42 smithi069 ceph-mon[21800]: from='mgr.14200 172.21.15.69:0/2429389701' entity='mgr.smithi069.vspncx' 2024-04-24T13:02:42.351 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:02:42 smithi069 ceph-mon[21800]: from='mgr.14200 172.21.15.69:0/2429389701' entity='mgr.smithi069.vspncx' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-24T13:02:42.351 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:02:42 smithi069 ceph-mon[21800]: from='mgr.14200 172.21.15.69:0/2429389701' entity='mgr.smithi069.vspncx' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-24T13:02:42.351 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:02:42 smithi069 ceph-mon[21800]: Deploying daemon mon.smithi114 on smithi114 2024-04-24T13:02:43.209 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-04-24T13:02:43.210 DEBUG:teuthology.orchestra.run.smithi114:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9150213e-023a-11ef-bc93-c7b262605968 -- ceph mon dump -f json 2024-04-24T13:02:46.100 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:02:45 smithi069 ceph-mon[21800]: from='mgr.14200 172.21.15.69:0/2429389701' entity='mgr.smithi069.vspncx' 2024-04-24T13:02:46.161 INFO:teuthology.orchestra.run.smithi114.stdout: 2024-04-24T13:02:46.161 INFO:teuthology.orchestra.run.smithi114.stdout:{"epoch":1,"fsid":"9150213e-023a-11ef-bc93-c7b262605968","modified":"2024-04-24T13:00:41.869875Z","created":"2024-04-24T13:00:41.869875Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi069","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.69:3300","nonce":0},{"type":"v1","addr":"172.21.15.69:6789","nonce":0}]},"addr":"172.21.15.69:6789/0","public_addr":"172.21.15.69:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-04-24T13:02:46.163 INFO:teuthology.orchestra.run.smithi114.stderr:dumped monmap epoch 1 2024-04-24T13:02:48.076 INFO:tasks.cephadm:Waiting for 2 mons in monmap... 2024-04-24T13:02:48.077 DEBUG:teuthology.orchestra.run.smithi114:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9150213e-023a-11ef-bc93-c7b262605968 -- ceph mon dump -f json 2024-04-24T13:02:52.101 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:02:51 smithi069 ceph-mon[21800]: from='mgr.14200 172.21.15.69:0/2429389701' entity='mgr.smithi069.vspncx' cmd=[{"prefix": "mon metadata", "id": "smithi069"}]: dispatch 2024-04-24T13:02:52.101 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:02:51 smithi069 ceph-mon[21800]: mon.smithi069 calling monitor election 2024-04-24T13:02:52.101 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:02:51 smithi069 ceph-mon[21800]: from='mgr.14200 172.21.15.69:0/2429389701' entity='mgr.smithi069.vspncx' cmd=[{"prefix": "mon metadata", "id": "smithi114"}]: dispatch 2024-04-24T13:02:52.101 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:02:51 smithi069 ceph-mon[21800]: from='mgr.14200 172.21.15.69:0/2429389701' entity='mgr.smithi069.vspncx' cmd=[{"prefix": "mon metadata", "id": "smithi114"}]: dispatch 2024-04-24T13:02:52.101 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:02:51 smithi069 ceph-mon[21800]: from='mgr.? 172.21.15.114:0/2326370075' entity='mgr.smithi114.oaurwi' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi114.oaurwi/crt"}]: dispatch 2024-04-24T13:02:52.101 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:02:51 smithi069 ceph-mon[21800]: from='mgr.14200 172.21.15.69:0/2429389701' entity='mgr.smithi069.vspncx' cmd=[{"prefix": "mon metadata", "id": "smithi114"}]: dispatch 2024-04-24T13:02:52.101 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:02:51 smithi069 ceph-mon[21800]: mon.smithi114 calling monitor election 2024-04-24T13:02:52.101 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:02:51 smithi069 ceph-mon[21800]: from='mgr.14200 172.21.15.69:0/2429389701' entity='mgr.smithi069.vspncx' cmd=[{"prefix": "mon metadata", "id": "smithi114"}]: dispatch 2024-04-24T13:02:52.101 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:02:51 smithi069 ceph-mon[21800]: pgmap v3: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-24T13:02:52.102 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:02:51 smithi069 ceph-mon[21800]: from='mgr.14200 172.21.15.69:0/2429389701' entity='mgr.smithi069.vspncx' cmd=[{"prefix": "mon metadata", "id": "smithi114"}]: dispatch 2024-04-24T13:02:52.102 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:02:51 smithi069 ceph-mon[21800]: from='mgr.14200 172.21.15.69:0/2429389701' entity='mgr.smithi069.vspncx' cmd=[{"prefix": "mon metadata", "id": "smithi114"}]: dispatch 2024-04-24T13:02:52.102 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:02:51 smithi069 ceph-mon[21800]: mon.smithi069 is new leader, mons smithi069,smithi114 in quorum (ranks 0,1) 2024-04-24T13:02:52.102 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:02:51 smithi069 ceph-mon[21800]: monmap e2: 2 mons at {smithi069=[v2:172.21.15.69:3300/0,v1:172.21.15.69:6789/0],smithi114=[v2:172.21.15.114:3300/0,v1:172.21.15.114:6789/0]} 2024-04-24T13:02:52.102 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:02:51 smithi069 ceph-mon[21800]: fsmap 2024-04-24T13:02:52.102 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:02:51 smithi069 ceph-mon[21800]: osdmap e5: 0 total, 0 up, 0 in 2024-04-24T13:02:52.102 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:02:51 smithi069 ceph-mon[21800]: mgrmap e17: smithi069.vspncx(active, since 21s) 2024-04-24T13:02:52.102 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:02:51 smithi069 ceph-mon[21800]: Standby manager daemon smithi114.oaurwi started 2024-04-24T13:02:52.102 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:02:51 smithi069 ceph-mon[21800]: from='mgr.? 172.21.15.114:0/2326370075' entity='mgr.smithi114.oaurwi' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-24T13:02:52.102 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:02:51 smithi069 ceph-mon[21800]: overall HEALTH_OK 2024-04-24T13:02:52.102 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:02:51 smithi069 ceph-mon[21800]: from='mgr.? 172.21.15.114:0/2326370075' entity='mgr.smithi114.oaurwi' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/smithi114.oaurwi/key"}]: dispatch 2024-04-24T13:02:52.103 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:02:51 smithi069 ceph-mon[21800]: from='mgr.? 172.21.15.114:0/2326370075' entity='mgr.smithi114.oaurwi' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-24T13:02:52.103 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:02:51 smithi069 ceph-mon[21800]: from='mgr.14200 172.21.15.69:0/2429389701' entity='mgr.smithi069.vspncx' 2024-04-24T13:02:52.880 INFO:teuthology.orchestra.run.smithi114.stdout: 2024-04-24T13:02:52.881 INFO:teuthology.orchestra.run.smithi114.stdout:{"epoch":2,"fsid":"9150213e-023a-11ef-bc93-c7b262605968","modified":"2024-04-24T13:02:46.562892Z","created":"2024-04-24T13:00:41.869875Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"smithi069","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.69:3300","nonce":0},{"type":"v1","addr":"172.21.15.69:6789","nonce":0}]},"addr":"172.21.15.69:6789/0","public_addr":"172.21.15.69:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":1,"name":"smithi114","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.114:3300","nonce":0},{"type":"v1","addr":"172.21.15.114:6789","nonce":0}]},"addr":"172.21.15.114:6789/0","public_addr":"172.21.15.114:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0,1]} 2024-04-24T13:02:52.881 INFO:teuthology.orchestra.run.smithi114.stderr:dumped monmap epoch 2 2024-04-24T13:02:53.100 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:02:52 smithi069 ceph-mon[21800]: Deploying daemon node-exporter.smithi114 on smithi114 2024-04-24T13:02:53.100 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:02:52 smithi069 ceph-mon[21800]: mgrmap e18: smithi069.vspncx(active, since 21s), standbys: smithi114.oaurwi 2024-04-24T13:02:53.101 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:02:52 smithi069 ceph-mon[21800]: from='mgr.14200 172.21.15.69:0/2429389701' entity='mgr.smithi069.vspncx' cmd=[{"prefix": "mgr metadata", "who": "smithi114.oaurwi", "id": "smithi114.oaurwi"}]: dispatch 2024-04-24T13:02:53.101 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:02:52 smithi069 ceph-mon[21800]: pgmap v4: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-24T13:02:53.101 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:02:52 smithi069 ceph-mon[21800]: from='mgr.14200 172.21.15.69:0/2429389701' entity='mgr.smithi069.vspncx' cmd=[{"prefix": "mon metadata", "id": "smithi114"}]: dispatch 2024-04-24T13:02:53.766 INFO:tasks.cephadm:Generating final ceph.conf file... 2024-04-24T13:02:53.766 DEBUG:teuthology.orchestra.run.smithi069:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9150213e-023a-11ef-bc93-c7b262605968 -- ceph config generate-minimal-conf 2024-04-24T13:02:54.079 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:02:53 smithi069 ceph-mon[21800]: from='client.? 172.21.15.114:0/1456492167' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-24T13:02:54.851 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:02:54 smithi069 ceph-mon[21800]: pgmap v5: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-24T13:02:55.682 INFO:teuthology.orchestra.run.smithi069.stdout:# minimal ceph.conf for 9150213e-023a-11ef-bc93-c7b262605968 2024-04-24T13:02:55.682 INFO:teuthology.orchestra.run.smithi069.stdout:[global] 2024-04-24T13:02:55.682 INFO:teuthology.orchestra.run.smithi069.stdout: fsid = 9150213e-023a-11ef-bc93-c7b262605968 2024-04-24T13:02:55.682 INFO:teuthology.orchestra.run.smithi069.stdout: mon_host = [v2:172.21.15.69:3300/0,v1:172.21.15.69:6789/0] [v2:172.21.15.114:3300/0,v1:172.21.15.114:6789/0] 2024-04-24T13:02:56.100 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:02:55 smithi069 ceph-mon[21800]: from='mgr.14200 172.21.15.69:0/2429389701' entity='mgr.smithi069.vspncx' 2024-04-24T13:02:56.100 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:02:55 smithi069 ceph-mon[21800]: from='mgr.14200 172.21.15.69:0/2429389701' entity='mgr.smithi069.vspncx' 2024-04-24T13:02:56.100 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:02:55 smithi069 ceph-mon[21800]: from='client.? 172.21.15.69:0/2059065759' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-24T13:02:56.192 INFO:journalctl@ceph.mon.smithi114.smithi114.stdout:Apr 24 13:02:55 smithi114 ceph-mon[23838]: from='client.? 172.21.15.69:0/2059065759' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-24T13:02:56.340 INFO:tasks.cephadm:Distributing (final) config and client.admin keyring... 2024-04-24T13:02:56.340 DEBUG:teuthology.orchestra.run.smithi069:> set -ex 2024-04-24T13:02:56.340 DEBUG:teuthology.orchestra.run.smithi069:> sudo dd of=/etc/ceph/ceph.conf 2024-04-24T13:02:56.365 DEBUG:teuthology.orchestra.run.smithi069:> set -ex 2024-04-24T13:02:56.366 DEBUG:teuthology.orchestra.run.smithi069:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2024-04-24T13:02:56.429 DEBUG:teuthology.orchestra.run.smithi114:> set -ex 2024-04-24T13:02:56.430 DEBUG:teuthology.orchestra.run.smithi114:> sudo dd of=/etc/ceph/ceph.conf 2024-04-24T13:02:56.458 DEBUG:teuthology.orchestra.run.smithi114:> set -ex 2024-04-24T13:02:56.458 DEBUG:teuthology.orchestra.run.smithi114:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2024-04-24T13:02:56.523 INFO:tasks.cephadm:Deploying OSDs... 2024-04-24T13:02:56.524 DEBUG:teuthology.orchestra.run.smithi069:> set -ex 2024-04-24T13:02:56.524 DEBUG:teuthology.orchestra.run.smithi069:> dd if=/scratch_devs of=/dev/stdout 2024-04-24T13:02:56.539 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2024-04-24T13:02:56.539 DEBUG:teuthology.orchestra.run.smithi069:> stat /dev/vg_nvme/lv_1 2024-04-24T13:02:56.594 INFO:teuthology.orchestra.run.smithi069.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2024-04-24T13:02:56.594 INFO:teuthology.orchestra.run.smithi069.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-04-24T13:02:56.594 INFO:teuthology.orchestra.run.smithi069.stdout:Device: 5h/5d Inode: 665 Links: 1 2024-04-24T13:02:56.594 INFO:teuthology.orchestra.run.smithi069.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-04-24T13:02:56.594 INFO:teuthology.orchestra.run.smithi069.stdout:Context: system_u:object_r:device_t:s0 2024-04-24T13:02:56.595 INFO:teuthology.orchestra.run.smithi069.stdout:Access: 2024-04-24 13:02:20.295316929 +0000 2024-04-24T13:02:56.595 INFO:teuthology.orchestra.run.smithi069.stdout:Modify: 2024-04-24 12:59:43.198342682 +0000 2024-04-24T13:02:56.595 INFO:teuthology.orchestra.run.smithi069.stdout:Change: 2024-04-24 12:59:43.198342682 +0000 2024-04-24T13:02:56.595 INFO:teuthology.orchestra.run.smithi069.stdout: Birth: 2024-04-24 12:59:43.198342682 +0000 2024-04-24T13:02:56.595 DEBUG:teuthology.orchestra.run.smithi069:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2024-04-24T13:02:56.658 INFO:teuthology.orchestra.run.smithi069.stderr:1+0 records in 2024-04-24T13:02:56.658 INFO:teuthology.orchestra.run.smithi069.stderr:1+0 records out 2024-04-24T13:02:56.658 INFO:teuthology.orchestra.run.smithi069.stderr:512 bytes copied, 0.000277524 s, 1.8 MB/s 2024-04-24T13:02:56.659 DEBUG:teuthology.orchestra.run.smithi069:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2024-04-24T13:02:56.715 DEBUG:teuthology.orchestra.run.smithi069:> stat /dev/vg_nvme/lv_2 2024-04-24T13:02:56.771 INFO:teuthology.orchestra.run.smithi069.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2024-04-24T13:02:56.771 INFO:teuthology.orchestra.run.smithi069.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-04-24T13:02:56.771 INFO:teuthology.orchestra.run.smithi069.stdout:Device: 5h/5d Inode: 686 Links: 1 2024-04-24T13:02:56.771 INFO:teuthology.orchestra.run.smithi069.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-04-24T13:02:56.771 INFO:teuthology.orchestra.run.smithi069.stdout:Context: system_u:object_r:device_t:s0 2024-04-24T13:02:56.771 INFO:teuthology.orchestra.run.smithi069.stdout:Access: 2024-04-24 13:02:20.295316929 +0000 2024-04-24T13:02:56.771 INFO:teuthology.orchestra.run.smithi069.stdout:Modify: 2024-04-24 12:59:43.569334275 +0000 2024-04-24T13:02:56.771 INFO:teuthology.orchestra.run.smithi069.stdout:Change: 2024-04-24 12:59:43.569334275 +0000 2024-04-24T13:02:56.771 INFO:teuthology.orchestra.run.smithi069.stdout: Birth: 2024-04-24 12:59:43.569334275 +0000 2024-04-24T13:02:56.772 DEBUG:teuthology.orchestra.run.smithi069:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2024-04-24T13:02:56.833 INFO:teuthology.orchestra.run.smithi069.stderr:1+0 records in 2024-04-24T13:02:56.834 INFO:teuthology.orchestra.run.smithi069.stderr:1+0 records out 2024-04-24T13:02:56.834 INFO:teuthology.orchestra.run.smithi069.stderr:512 bytes copied, 0.000175908 s, 2.9 MB/s 2024-04-24T13:02:56.835 DEBUG:teuthology.orchestra.run.smithi069:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2024-04-24T13:02:56.890 DEBUG:teuthology.orchestra.run.smithi069:> stat /dev/vg_nvme/lv_3 2024-04-24T13:02:56.944 INFO:teuthology.orchestra.run.smithi069.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2024-04-24T13:02:56.945 INFO:teuthology.orchestra.run.smithi069.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-04-24T13:02:56.945 INFO:teuthology.orchestra.run.smithi069.stdout:Device: 5h/5d Inode: 705 Links: 1 2024-04-24T13:02:56.945 INFO:teuthology.orchestra.run.smithi069.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-04-24T13:02:56.945 INFO:teuthology.orchestra.run.smithi069.stdout:Context: system_u:object_r:device_t:s0 2024-04-24T13:02:56.945 INFO:teuthology.orchestra.run.smithi069.stdout:Access: 2024-04-24 13:02:20.296316907 +0000 2024-04-24T13:02:56.945 INFO:teuthology.orchestra.run.smithi069.stdout:Modify: 2024-04-24 12:59:43.939325890 +0000 2024-04-24T13:02:56.945 INFO:teuthology.orchestra.run.smithi069.stdout:Change: 2024-04-24 12:59:43.939325890 +0000 2024-04-24T13:02:56.945 INFO:teuthology.orchestra.run.smithi069.stdout: Birth: 2024-04-24 12:59:43.939325890 +0000 2024-04-24T13:02:56.945 DEBUG:teuthology.orchestra.run.smithi069:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2024-04-24T13:02:57.004 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:02:56 smithi069 ceph-mon[21800]: Deploying daemon prometheus.smithi069 on smithi069 2024-04-24T13:02:57.004 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:02:56 smithi069 ceph-mon[21800]: pgmap v6: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-24T13:02:57.006 INFO:teuthology.orchestra.run.smithi069.stderr:1+0 records in 2024-04-24T13:02:57.007 INFO:teuthology.orchestra.run.smithi069.stderr:1+0 records out 2024-04-24T13:02:57.007 INFO:teuthology.orchestra.run.smithi069.stderr:512 bytes copied, 0.000176249 s, 2.9 MB/s 2024-04-24T13:02:57.008 DEBUG:teuthology.orchestra.run.smithi069:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2024-04-24T13:02:57.063 DEBUG:teuthology.orchestra.run.smithi069:> stat /dev/vg_nvme/lv_4 2024-04-24T13:02:57.117 INFO:teuthology.orchestra.run.smithi069.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2024-04-24T13:02:57.117 INFO:teuthology.orchestra.run.smithi069.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-04-24T13:02:57.118 INFO:teuthology.orchestra.run.smithi069.stdout:Device: 5h/5d Inode: 721 Links: 1 2024-04-24T13:02:57.118 INFO:teuthology.orchestra.run.smithi069.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-04-24T13:02:57.118 INFO:teuthology.orchestra.run.smithi069.stdout:Context: system_u:object_r:device_t:s0 2024-04-24T13:02:57.118 INFO:teuthology.orchestra.run.smithi069.stdout:Access: 2024-04-24 13:02:20.296316907 +0000 2024-04-24T13:02:57.118 INFO:teuthology.orchestra.run.smithi069.stdout:Modify: 2024-04-24 12:59:44.292317890 +0000 2024-04-24T13:02:57.118 INFO:teuthology.orchestra.run.smithi069.stdout:Change: 2024-04-24 12:59:44.292317890 +0000 2024-04-24T13:02:57.118 INFO:teuthology.orchestra.run.smithi069.stdout: Birth: 2024-04-24 12:59:44.292317890 +0000 2024-04-24T13:02:57.118 DEBUG:teuthology.orchestra.run.smithi069:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2024-04-24T13:02:57.180 INFO:teuthology.orchestra.run.smithi069.stderr:1+0 records in 2024-04-24T13:02:57.180 INFO:teuthology.orchestra.run.smithi069.stderr:1+0 records out 2024-04-24T13:02:57.180 INFO:teuthology.orchestra.run.smithi069.stderr:512 bytes copied, 0.000187955 s, 2.7 MB/s 2024-04-24T13:02:57.182 DEBUG:teuthology.orchestra.run.smithi069:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2024-04-24T13:02:57.192 INFO:journalctl@ceph.mon.smithi114.smithi114.stdout:Apr 24 13:02:56 smithi114 ceph-mon[23838]: Deploying daemon prometheus.smithi069 on smithi069 2024-04-24T13:02:57.192 INFO:journalctl@ceph.mon.smithi114.smithi114.stdout:Apr 24 13:02:56 smithi114 ceph-mon[23838]: pgmap v6: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-24T13:02:57.237 DEBUG:teuthology.orchestra.run.smithi114:> set -ex 2024-04-24T13:02:57.237 DEBUG:teuthology.orchestra.run.smithi114:> dd if=/scratch_devs of=/dev/stdout 2024-04-24T13:02:57.256 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2024-04-24T13:02:57.256 DEBUG:teuthology.orchestra.run.smithi114:> stat /dev/vg_nvme/lv_1 2024-04-24T13:02:57.314 INFO:teuthology.orchestra.run.smithi114.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2024-04-24T13:02:57.314 INFO:teuthology.orchestra.run.smithi114.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-04-24T13:02:57.314 INFO:teuthology.orchestra.run.smithi114.stdout:Device: 5h/5d Inode: 666 Links: 1 2024-04-24T13:02:57.314 INFO:teuthology.orchestra.run.smithi114.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-04-24T13:02:57.314 INFO:teuthology.orchestra.run.smithi114.stdout:Context: system_u:object_r:device_t:s0 2024-04-24T13:02:57.314 INFO:teuthology.orchestra.run.smithi114.stdout:Access: 2024-04-24 13:02:52.137298223 +0000 2024-04-24T13:02:57.314 INFO:teuthology.orchestra.run.smithi114.stdout:Modify: 2024-04-24 12:59:43.151511376 +0000 2024-04-24T13:02:57.314 INFO:teuthology.orchestra.run.smithi114.stdout:Change: 2024-04-24 12:59:43.151511376 +0000 2024-04-24T13:02:57.315 INFO:teuthology.orchestra.run.smithi114.stdout: Birth: 2024-04-24 12:59:43.151511376 +0000 2024-04-24T13:02:57.315 DEBUG:teuthology.orchestra.run.smithi114:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2024-04-24T13:02:57.381 INFO:teuthology.orchestra.run.smithi114.stderr:1+0 records in 2024-04-24T13:02:57.381 INFO:teuthology.orchestra.run.smithi114.stderr:1+0 records out 2024-04-24T13:02:57.381 INFO:teuthology.orchestra.run.smithi114.stderr:512 bytes copied, 0.000275002 s, 1.9 MB/s 2024-04-24T13:02:57.382 DEBUG:teuthology.orchestra.run.smithi114:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2024-04-24T13:02:57.440 DEBUG:teuthology.orchestra.run.smithi114:> stat /dev/vg_nvme/lv_2 2024-04-24T13:02:57.497 INFO:teuthology.orchestra.run.smithi114.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2024-04-24T13:02:57.497 INFO:teuthology.orchestra.run.smithi114.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-04-24T13:02:57.497 INFO:teuthology.orchestra.run.smithi114.stdout:Device: 5h/5d Inode: 683 Links: 1 2024-04-24T13:02:57.497 INFO:teuthology.orchestra.run.smithi114.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-04-24T13:02:57.497 INFO:teuthology.orchestra.run.smithi114.stdout:Context: system_u:object_r:device_t:s0 2024-04-24T13:02:57.497 INFO:teuthology.orchestra.run.smithi114.stdout:Access: 2024-04-24 13:02:52.137298223 +0000 2024-04-24T13:02:57.497 INFO:teuthology.orchestra.run.smithi114.stdout:Modify: 2024-04-24 12:59:43.482503868 +0000 2024-04-24T13:02:57.497 INFO:teuthology.orchestra.run.smithi114.stdout:Change: 2024-04-24 12:59:43.482503868 +0000 2024-04-24T13:02:57.497 INFO:teuthology.orchestra.run.smithi114.stdout: Birth: 2024-04-24 12:59:43.482503868 +0000 2024-04-24T13:02:57.497 DEBUG:teuthology.orchestra.run.smithi114:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2024-04-24T13:02:57.562 INFO:teuthology.orchestra.run.smithi114.stderr:1+0 records in 2024-04-24T13:02:57.562 INFO:teuthology.orchestra.run.smithi114.stderr:1+0 records out 2024-04-24T13:02:57.562 INFO:teuthology.orchestra.run.smithi114.stderr:512 bytes copied, 0.000197168 s, 2.6 MB/s 2024-04-24T13:02:57.563 DEBUG:teuthology.orchestra.run.smithi114:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2024-04-24T13:02:57.620 DEBUG:teuthology.orchestra.run.smithi114:> stat /dev/vg_nvme/lv_3 2024-04-24T13:02:57.676 INFO:teuthology.orchestra.run.smithi114.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2024-04-24T13:02:57.676 INFO:teuthology.orchestra.run.smithi114.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-04-24T13:02:57.677 INFO:teuthology.orchestra.run.smithi114.stdout:Device: 5h/5d Inode: 702 Links: 1 2024-04-24T13:02:57.677 INFO:teuthology.orchestra.run.smithi114.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-04-24T13:02:57.677 INFO:teuthology.orchestra.run.smithi114.stdout:Context: system_u:object_r:device_t:s0 2024-04-24T13:02:57.677 INFO:teuthology.orchestra.run.smithi114.stdout:Access: 2024-04-24 13:02:52.138298201 +0000 2024-04-24T13:02:57.677 INFO:teuthology.orchestra.run.smithi114.stdout:Modify: 2024-04-24 12:59:43.824496109 +0000 2024-04-24T13:02:57.677 INFO:teuthology.orchestra.run.smithi114.stdout:Change: 2024-04-24 12:59:43.824496109 +0000 2024-04-24T13:02:57.677 INFO:teuthology.orchestra.run.smithi114.stdout: Birth: 2024-04-24 12:59:43.824496109 +0000 2024-04-24T13:02:57.677 DEBUG:teuthology.orchestra.run.smithi114:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2024-04-24T13:02:57.740 INFO:teuthology.orchestra.run.smithi114.stderr:1+0 records in 2024-04-24T13:02:57.740 INFO:teuthology.orchestra.run.smithi114.stderr:1+0 records out 2024-04-24T13:02:57.740 INFO:teuthology.orchestra.run.smithi114.stderr:512 bytes copied, 0.0002342 s, 2.2 MB/s 2024-04-24T13:02:57.742 DEBUG:teuthology.orchestra.run.smithi114:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2024-04-24T13:02:57.799 DEBUG:teuthology.orchestra.run.smithi114:> stat /dev/vg_nvme/lv_4 2024-04-24T13:02:57.856 INFO:teuthology.orchestra.run.smithi114.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2024-04-24T13:02:57.856 INFO:teuthology.orchestra.run.smithi114.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-04-24T13:02:57.856 INFO:teuthology.orchestra.run.smithi114.stdout:Device: 5h/5d Inode: 723 Links: 1 2024-04-24T13:02:57.856 INFO:teuthology.orchestra.run.smithi114.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-04-24T13:02:57.857 INFO:teuthology.orchestra.run.smithi114.stdout:Context: system_u:object_r:device_t:s0 2024-04-24T13:02:57.857 INFO:teuthology.orchestra.run.smithi114.stdout:Access: 2024-04-24 13:02:52.138298201 +0000 2024-04-24T13:02:57.857 INFO:teuthology.orchestra.run.smithi114.stdout:Modify: 2024-04-24 12:59:44.399483066 +0000 2024-04-24T13:02:57.857 INFO:teuthology.orchestra.run.smithi114.stdout:Change: 2024-04-24 12:59:44.399483066 +0000 2024-04-24T13:02:57.857 INFO:teuthology.orchestra.run.smithi114.stdout: Birth: 2024-04-24 12:59:44.399483066 +0000 2024-04-24T13:02:57.858 DEBUG:teuthology.orchestra.run.smithi114:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2024-04-24T13:02:57.921 INFO:teuthology.orchestra.run.smithi114.stderr:1+0 records in 2024-04-24T13:02:57.921 INFO:teuthology.orchestra.run.smithi114.stderr:1+0 records out 2024-04-24T13:02:57.921 INFO:teuthology.orchestra.run.smithi114.stderr:512 bytes copied, 0.000218078 s, 2.3 MB/s 2024-04-24T13:02:57.922 DEBUG:teuthology.orchestra.run.smithi114:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2024-04-24T13:02:57.982 INFO:tasks.cephadm:Deploying osd.0 on smithi069 with /dev/vg_nvme/lv_4... 2024-04-24T13:02:57.982 DEBUG:teuthology.orchestra.run.smithi069:> sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9150213e-023a-11ef-bc93-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_4 2024-04-24T13:02:59.100 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:02:58 smithi069 ceph-mon[21800]: pgmap v7: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-24T13:02:59.192 INFO:journalctl@ceph.mon.smithi114.smithi114.stdout:Apr 24 13:02:58 smithi114 ceph-mon[23838]: pgmap v7: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-24T13:03:01.100 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:03:00 smithi069 ceph-mon[21800]: pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-24T13:03:01.100 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:03:00 smithi069 ceph-mon[21800]: from='mgr.14200 172.21.15.69:0/2429389701' entity='mgr.smithi069.vspncx' 2024-04-24T13:03:01.192 INFO:journalctl@ceph.mon.smithi114.smithi114.stdout:Apr 24 13:03:00 smithi114 ceph-mon[23838]: pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-24T13:03:01.192 INFO:journalctl@ceph.mon.smithi114.smithi114.stdout:Apr 24 13:03:00 smithi114 ceph-mon[23838]: from='mgr.14200 172.21.15.69:0/2429389701' entity='mgr.smithi069.vspncx' 2024-04-24T13:03:02.350 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:03:02 smithi069 ceph-mon[21800]: pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-24T13:03:02.442 INFO:journalctl@ceph.mon.smithi114.smithi114.stdout:Apr 24 13:03:02 smithi114 ceph-mon[23838]: pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-24T13:03:04.192 INFO:journalctl@ceph.mon.smithi114.smithi114.stdout:Apr 24 13:03:03 smithi114 ceph-mon[23838]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-24T13:03:04.291 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:03:03 smithi069 ceph-mon[21800]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-24T13:03:06.100 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:03:05 smithi069 ceph-mon[21800]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-24T13:03:06.192 INFO:journalctl@ceph.mon.smithi114.smithi114.stdout:Apr 24 13:03:05 smithi114 ceph-mon[23838]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-24T13:03:08.350 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:03:08 smithi069 ceph-mon[21800]: from='mgr.14200 172.21.15.69:0/2429389701' entity='mgr.smithi069.vspncx' 2024-04-24T13:03:08.350 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:03:08 smithi069 ceph-mon[21800]: from='mgr.14200 172.21.15.69:0/2429389701' entity='mgr.smithi069.vspncx' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-24T13:03:08.350 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:03:08 smithi069 ceph-mon[21800]: from='mgr.14200 172.21.15.69:0/2429389701' entity='mgr.smithi069.vspncx' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-24T13:03:08.351 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:03:08 smithi069 ceph-mon[21800]: from='mgr.14200 172.21.15.69:0/2429389701' entity='mgr.smithi069.vspncx' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-24T13:03:08.351 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:03:08 smithi069 ceph-mon[21800]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-24T13:03:08.410 INFO:journalctl@ceph.mon.smithi114.smithi114.stdout:Apr 24 13:03:08 smithi114 ceph-mon[23838]: from='mgr.14200 172.21.15.69:0/2429389701' entity='mgr.smithi069.vspncx' 2024-04-24T13:03:08.411 INFO:journalctl@ceph.mon.smithi114.smithi114.stdout:Apr 24 13:03:08 smithi114 ceph-mon[23838]: from='mgr.14200 172.21.15.69:0/2429389701' entity='mgr.smithi069.vspncx' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-24T13:03:08.411 INFO:journalctl@ceph.mon.smithi114.smithi114.stdout:Apr 24 13:03:08 smithi114 ceph-mon[23838]: from='mgr.14200 172.21.15.69:0/2429389701' entity='mgr.smithi069.vspncx' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-24T13:03:08.411 INFO:journalctl@ceph.mon.smithi114.smithi114.stdout:Apr 24 13:03:08 smithi114 ceph-mon[23838]: from='mgr.14200 172.21.15.69:0/2429389701' entity='mgr.smithi069.vspncx' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-24T13:03:08.411 INFO:journalctl@ceph.mon.smithi114.smithi114.stdout:Apr 24 13:03:08 smithi114 ceph-mon[23838]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-24T13:03:09.892 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:03:09 smithi069 ceph-mon[21800]: from='mgr.14200 172.21.15.69:0/2429389701' entity='mgr.smithi069.vspncx' 2024-04-24T13:03:09.893 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:03:09 smithi069 ceph-mon[21800]: Updating smithi114:/etc/ceph/ceph.conf 2024-04-24T13:03:09.893 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:03:09 smithi069 ceph-mon[21800]: Updating smithi114:/etc/ceph/ceph.client.admin.keyring 2024-04-24T13:03:09.893 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:03:09 smithi069 ceph-mon[21800]: from='mgr.14200 172.21.15.69:0/2429389701' entity='mgr.smithi069.vspncx' 2024-04-24T13:03:09.893 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:03:09 smithi069 ceph-mon[21800]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-24T13:03:09.893 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:03:09 smithi069 ceph-mon[21800]: from='mgr.14200 172.21.15.69:0/2429389701' entity='mgr.smithi069.vspncx' 2024-04-24T13:03:09.895 INFO:teuthology.orchestra.run.smithi069.stderr:Non-zero exit code 2 from /bin/podman run --rm --ipc=host --stop-signal=SIGTERM --net=host --entrypoint /usr/sbin/ceph-volume --privileged --group-add=disk --init -e CONTAINER_IMAGE=quay.io/ceph/ceph:v17.2.0 -e NODE_NAME=smithi069 -e CEPH_USE_RANDOM_NONCE=1 -e CEPH_VOLUME_SKIP_RESTORECON=yes -e CEPH_VOLUME_DEBUG=1 -v /var/run/ceph/9150213e-023a-11ef-bc93-c7b262605968:/var/run/ceph:z -v /var/log/ceph/9150213e-023a-11ef-bc93-c7b262605968:/var/log/ceph:z -v /var/lib/ceph/9150213e-023a-11ef-bc93-c7b262605968/crash:/var/lib/ceph/crash:z -v /run/systemd/journal:/run/systemd/journal -v /dev:/dev -v /run/udev:/run/udev -v /sys:/sys -v /run/lvm:/run/lvm -v /run/lock/lvm:/run/lock/lvm -v /var/lib/ceph/9150213e-023a-11ef-bc93-c7b262605968/selinux:/sys/fs/selinux:ro -v /:/rootfs -v /tmp/ceph-tmpto0dvgly:/etc/ceph/ceph.conf:z -v /tmp/ceph-tmpqy83_v9_:/var/lib/ceph/bootstrap-osd/ceph.keyring:z quay.io/ceph/ceph:v17.2.0 ceph-volume lvm zap /dev/vg_nvme/lv_4 2024-04-24T13:03:09.895 INFO:teuthology.orchestra.run.smithi069.stderr:/bin/podman: stderr usage: ceph-volume [-h] [--cluster CLUSTER] 2024-04-24T13:03:09.895 INFO:teuthology.orchestra.run.smithi069.stderr:/bin/podman: stderr [--log-level {debug,info,warning,error,critical}] 2024-04-24T13:03:09.895 INFO:teuthology.orchestra.run.smithi069.stderr:/bin/podman: stderr [--log-path LOG_PATH] 2024-04-24T13:03:09.895 INFO:teuthology.orchestra.run.smithi069.stderr:/bin/podman: stderr ceph-volume: error: unrecognized arguments: ceph-volume 2024-04-24T13:03:09.897 INFO:teuthology.orchestra.run.smithi069.stderr:Traceback (most recent call last): 2024-04-24T13:03:09.897 INFO:teuthology.orchestra.run.smithi069.stderr: File "/home/ubuntu/cephtest/cephadm", line 8634, in 2024-04-24T13:03:09.898 INFO:teuthology.orchestra.run.smithi069.stderr: main() 2024-04-24T13:03:09.898 INFO:teuthology.orchestra.run.smithi069.stderr: File "/home/ubuntu/cephtest/cephadm", line 8622, in main 2024-04-24T13:03:09.899 INFO:teuthology.orchestra.run.smithi069.stderr: r = ctx.func(ctx) 2024-04-24T13:03:09.899 INFO:teuthology.orchestra.run.smithi069.stderr: File "/home/ubuntu/cephtest/cephadm", line 1870, in _infer_config 2024-04-24T13:03:09.900 INFO:teuthology.orchestra.run.smithi069.stderr: return func(ctx) 2024-04-24T13:03:09.900 INFO:teuthology.orchestra.run.smithi069.stderr: File "/home/ubuntu/cephtest/cephadm", line 1827, in _infer_fsid 2024-04-24T13:03:09.900 INFO:teuthology.orchestra.run.smithi069.stderr: return func(ctx) 2024-04-24T13:03:09.900 INFO:teuthology.orchestra.run.smithi069.stderr: File "/home/ubuntu/cephtest/cephadm", line 1914, in _infer_image 2024-04-24T13:03:09.900 INFO:teuthology.orchestra.run.smithi069.stderr: return func(ctx) 2024-04-24T13:03:09.900 INFO:teuthology.orchestra.run.smithi069.stderr: File "/home/ubuntu/cephtest/cephadm", line 1814, in _validate_fsid 2024-04-24T13:03:09.900 INFO:teuthology.orchestra.run.smithi069.stderr: return func(ctx) 2024-04-24T13:03:09.901 INFO:teuthology.orchestra.run.smithi069.stderr: File "/home/ubuntu/cephtest/cephadm", line 5562, in command_ceph_volume 2024-04-24T13:03:09.901 INFO:teuthology.orchestra.run.smithi069.stderr: out, err, code = call_throws(ctx, c.run_cmd()) 2024-04-24T13:03:09.901 INFO:teuthology.orchestra.run.smithi069.stderr: File "/home/ubuntu/cephtest/cephadm", line 1616, in call_throws 2024-04-24T13:03:09.901 INFO:teuthology.orchestra.run.smithi069.stderr: raise RuntimeError('Failed command: %s' % ' '.join(command)) 2024-04-24T13:03:09.902 INFO:teuthology.orchestra.run.smithi069.stderr:RuntimeError: Failed command: /bin/podman run --rm --ipc=host --stop-signal=SIGTERM --net=host --entrypoint /usr/sbin/ceph-volume --privileged --group-add=disk --init -e CONTAINER_IMAGE=quay.io/ceph/ceph:v17.2.0 -e NODE_NAME=smithi069 -e CEPH_USE_RANDOM_NONCE=1 -e CEPH_VOLUME_SKIP_RESTORECON=yes -e CEPH_VOLUME_DEBUG=1 -v /var/run/ceph/9150213e-023a-11ef-bc93-c7b262605968:/var/run/ceph:z -v /var/log/ceph/9150213e-023a-11ef-bc93-c7b262605968:/var/log/ceph:z -v /var/lib/ceph/9150213e-023a-11ef-bc93-c7b262605968/crash:/var/lib/ceph/crash:z -v /run/systemd/journal:/run/systemd/journal -v /dev:/dev -v /run/udev:/run/udev -v /sys:/sys -v /run/lvm:/run/lvm -v /run/lock/lvm:/run/lock/lvm -v /var/lib/ceph/9150213e-023a-11ef-bc93-c7b262605968/selinux:/sys/fs/selinux:ro -v /:/rootfs -v /tmp/ceph-tmpto0dvgly:/etc/ceph/ceph.conf:z -v /tmp/ceph-tmpqy83_v9_:/var/lib/ceph/bootstrap-osd/ceph.keyring:z quay.io/ceph/ceph:v17.2.0 ceph-volume lvm zap /dev/vg_nvme/lv_4 2024-04-24T13:03:09.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-24T13:03:09.922 ERROR:teuthology.contextutil:Saw exception from nested tasks Traceback (most recent call last): File "/home/teuthworker/src/git.ceph.com_teuthology_9481b1d62f50e7d0a4f3dd83adf6945b08d5ff17/teuthology/contextutil.py", line 30, in nested vars.append(enter()) File "/usr/lib/python3.8/contextlib.py", line 113, in __enter__ return next(self.gen) File "/home/teuthworker/src/git.ceph.com_ceph-c_23fcfb96e7e1a49d12a94e3f87a8e3f06db2a1ec/qa/tasks/cephadm.py", line 1091, in ceph_osds remote.run( File "/home/teuthworker/src/git.ceph.com_teuthology_9481b1d62f50e7d0a4f3dd83adf6945b08d5ff17/teuthology/orchestra/remote.py", line 523, in run r = self._runner(client=self.ssh, name=self.shortname, **kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_9481b1d62f50e7d0a4f3dd83adf6945b08d5ff17/teuthology/orchestra/run.py", line 455, in run r.wait() File "/home/teuthworker/src/git.ceph.com_teuthology_9481b1d62f50e7d0a4f3dd83adf6945b08d5ff17/teuthology/orchestra/run.py", line 161, in wait self._raise_for_status() File "/home/teuthworker/src/git.ceph.com_teuthology_9481b1d62f50e7d0a4f3dd83adf6945b08d5ff17/teuthology/orchestra/run.py", line 181, in _raise_for_status raise CommandFailedError( teuthology.exceptions.CommandFailedError: Command failed on smithi069 with status 1: 'sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9150213e-023a-11ef-bc93-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_4' 2024-04-24T13:03:09.924 DEBUG:teuthology.orchestra.run.smithi069:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-04-24T13:03:09.954 DEBUG:teuthology.orchestra.run.smithi114:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-04-24T13:03:09.979 INFO:tasks.cephadm:Cleaning up testdir ceph.* files... 2024-04-24T13:03:09.979 DEBUG:teuthology.orchestra.run.smithi069:> rm -f /home/ubuntu/cephtest/seed.ceph.conf /home/ubuntu/cephtest/ceph.pub 2024-04-24T13:03:10.012 DEBUG:teuthology.orchestra.run.smithi114:> rm -f /home/ubuntu/cephtest/seed.ceph.conf /home/ubuntu/cephtest/ceph.pub 2024-04-24T13:03:10.037 INFO:tasks.cephadm:Stopping all daemons... 2024-04-24T13:03:10.037 INFO:tasks.cephadm.mon.smithi069:Stopping mon.smithi069... 2024-04-24T13:03:10.037 DEBUG:teuthology.orchestra.run.smithi069:> sudo systemctl stop ceph-9150213e-023a-11ef-bc93-c7b262605968@mon.smithi069 2024-04-24T13:03:10.192 INFO:journalctl@ceph.mon.smithi114.smithi114.stdout:Apr 24 13:03:09 smithi114 ceph-mon[23838]: from='mgr.14200 172.21.15.69:0/2429389701' entity='mgr.smithi069.vspncx' 2024-04-24T13:03:10.193 INFO:journalctl@ceph.mon.smithi114.smithi114.stdout:Apr 24 13:03:09 smithi114 ceph-mon[23838]: Updating smithi114:/etc/ceph/ceph.conf 2024-04-24T13:03:10.193 INFO:journalctl@ceph.mon.smithi114.smithi114.stdout:Apr 24 13:03:09 smithi114 ceph-mon[23838]: Updating smithi114:/etc/ceph/ceph.client.admin.keyring 2024-04-24T13:03:10.193 INFO:journalctl@ceph.mon.smithi114.smithi114.stdout:Apr 24 13:03:09 smithi114 ceph-mon[23838]: from='mgr.14200 172.21.15.69:0/2429389701' entity='mgr.smithi069.vspncx' 2024-04-24T13:03:10.193 INFO:journalctl@ceph.mon.smithi114.smithi114.stdout:Apr 24 13:03:09 smithi114 ceph-mon[23838]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-24T13:03:10.193 INFO:journalctl@ceph.mon.smithi114.smithi114.stdout:Apr 24 13:03:09 smithi114 ceph-mon[23838]: from='mgr.14200 172.21.15.69:0/2429389701' entity='mgr.smithi069.vspncx' 2024-04-24T13:03:10.296 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:03:10 smithi069 systemd[1]: Stopping Ceph mon.smithi069 for 9150213e-023a-11ef-bc93-c7b262605968... 2024-04-24T13:03:10.600 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:03:10 smithi069 bash[29578]: Error: no container with name or ID "ceph-9150213e-023a-11ef-bc93-c7b262605968-mon.smithi069" found: no such container 2024-04-24T13:03:10.600 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:03:10 smithi069 ceph-9150213e-023a-11ef-bc93-c7b262605968-mon-smithi069[21796]: 2024-04-24T13:03:10.415+0000 7fe5d661d700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.smithi069 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-04-24T13:03:10.600 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:03:10 smithi069 ceph-9150213e-023a-11ef-bc93-c7b262605968-mon-smithi069[21796]: 2024-04-24T13:03:10.415+0000 7fe5d661d700 -1 mon.smithi069@0(leader) e2 *** Got Signal Terminated *** 2024-04-24T13:03:11.065 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:03:10 smithi069 podman[29607]: 2024-04-24 13:03:10.798296489 +0000 UTC m=+0.490070093 container died 82aada682e0492920095b94da619855b8ceb3ca28c06247cbdccba51b632f64b (image=quay.io/ceph/ceph:v17.2.0, name=ceph-9150213e-023a-11ef-bc93-c7b262605968-mon-smithi069, name=centos-stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.component=centos-stream-container, GIT_REPO=https://github.com/ceph/ceph-container.git, version=8, RELEASE=HEAD, architecture=x86_64, io.buildah.version=1.19.8, ceph=True, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, vendor=Red Hat, Inc., distribution-scope=public, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, build-date=2022-05-03T08:36:31.336870, GIT_CLEAN=True, release=754, GIT_BRANCH=HEAD, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, maintainer=Guillaume Abrioux , io.k8s.display-name=CentOS Stream 8, io.openshift.tags=base centos centos-stream, io.openshift.expose-services=, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, CEPH_POINT_RELEASE=-17.2.0, vcs-type=git) 2024-04-24T13:03:11.332 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:03:11 smithi069 podman[29607]: 2024-04-24 13:03:11.166743334 +0000 UTC m=+0.858516940 container cleanup 82aada682e0492920095b94da619855b8ceb3ca28c06247cbdccba51b632f64b (image=quay.io/ceph/ceph:v17.2.0, name=ceph-9150213e-023a-11ef-bc93-c7b262605968-mon-smithi069, CEPH_POINT_RELEASE=-17.2.0, vcs-type=git, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, io.openshift.tags=base centos centos-stream, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, distribution-scope=public, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, architecture=x86_64, release=754, io.buildah.version=1.19.8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.expose-services=, version=8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_CLEAN=True, ceph=True, io.k8s.display-name=CentOS Stream 8, build-date=2022-05-03T08:36:31.336870, RELEASE=HEAD, com.redhat.component=centos-stream-container, GIT_BRANCH=HEAD, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, name=centos-stream, maintainer=Guillaume Abrioux , vendor=Red Hat, Inc., GIT_REPO=https://github.com/ceph/ceph-container.git) 2024-04-24T13:03:11.332 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:03:11 smithi069 bash[29607]: ceph-9150213e-023a-11ef-bc93-c7b262605968-mon-smithi069 2024-04-24T13:03:11.600 INFO:journalctl@ceph.mon.smithi069.smithi069.stdout:Apr 24 13:03:11 smithi069 podman[29666]: 2024-04-24 13:03:11.334585445 +0000 UTC m=+0.762675402 container remove 82aada682e0492920095b94da619855b8ceb3ca28c06247cbdccba51b632f64b (image=quay.io/ceph/ceph:v17.2.0, name=ceph-9150213e-023a-11ef-bc93-c7b262605968-mon-smithi069, com.redhat.license_terms=https://centos.org/legal/licensing-policy/, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, architecture=x86_64, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, RELEASE=HEAD, GIT_REPO=https://github.com/ceph/ceph-container.git, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, distribution-scope=public, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, version=8, CEPH_POINT_RELEASE=-17.2.0, GIT_BRANCH=HEAD, release=754, name=centos-stream, build-date=2022-05-03T08:36:31.336870, GIT_CLEAN=True, io.openshift.expose-services=, maintainer=Guillaume Abrioux , ceph=True, com.redhat.component=centos-stream-container, vcs-type=git, io.buildah.version=1.19.8, vendor=Red Hat, Inc., io.k8s.display-name=CentOS Stream 8, io.openshift.tags=base centos centos-stream) 2024-04-24T13:03:11.642 DEBUG:teuthology.orchestra.run.smithi069:> sudo pkill -f 'journalctl -f -n 0 -u ceph-9150213e-023a-11ef-bc93-c7b262605968@mon.smithi069.service' 2024-04-24T13:03:11.688 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-24T13:03:11.688 INFO:tasks.cephadm.mon.smithi069:Stopped mon.smithi069 2024-04-24T13:03:11.689 INFO:tasks.cephadm.mon.smithi114:Stopping mon.smithi114... 2024-04-24T13:03:11.689 DEBUG:teuthology.orchestra.run.smithi114:> sudo systemctl stop ceph-9150213e-023a-11ef-bc93-c7b262605968@mon.smithi114 2024-04-24T13:03:12.033 INFO:journalctl@ceph.mon.smithi114.smithi114.stdout:Apr 24 13:03:11 smithi114 systemd[1]: Stopping Ceph mon.smithi114 for 9150213e-023a-11ef-bc93-c7b262605968... 2024-04-24T13:03:12.034 INFO:journalctl@ceph.mon.smithi114.smithi114.stdout:Apr 24 13:03:11 smithi114 bash[25112]: Error: no container with name or ID "ceph-9150213e-023a-11ef-bc93-c7b262605968-mon.smithi114" found: no such container 2024-04-24T13:03:12.034 INFO:journalctl@ceph.mon.smithi114.smithi114.stdout:Apr 24 13:03:11 smithi114 ceph-9150213e-023a-11ef-bc93-c7b262605968-mon-smithi114[23818]: 2024-04-24T13:03:11.901+0000 7fca5b833700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.smithi114 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-04-24T13:03:12.034 INFO:journalctl@ceph.mon.smithi114.smithi114.stdout:Apr 24 13:03:11 smithi114 ceph-9150213e-023a-11ef-bc93-c7b262605968-mon-smithi114[23818]: 2024-04-24T13:03:11.901+0000 7fca5b833700 -1 mon.smithi114@1(peon) e2 *** Got Signal Terminated *** 2024-04-24T13:03:12.300 INFO:journalctl@ceph.mon.smithi114.smithi114.stdout:Apr 24 13:03:12 smithi114 podman[25124]: 2024-04-24 13:03:12.035861463 +0000 UTC m=+0.259798418 container died 73aec54b7d5c8b66d5e4a8d120954329b1c24315a600019adbb88cf8191cd9c5 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-9150213e-023a-11ef-bc93-c7b262605968-mon-smithi114, maintainer=Guillaume Abrioux , vcs-type=git, RELEASE=HEAD, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, io.openshift.tags=base centos centos-stream, distribution-scope=public, GIT_CLEAN=True, name=centos-stream, GIT_REPO=https://github.com/ceph/ceph-container.git, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, vendor=Red Hat, Inc., ceph=True, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, version=8, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, com.redhat.component=centos-stream-container, release=754, CEPH_POINT_RELEASE=-17.2.0, io.openshift.expose-services=, GIT_BRANCH=HEAD, build-date=2022-05-03T08:36:31.336870, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, architecture=x86_64, io.buildah.version=1.19.8, io.k8s.display-name=CentOS Stream 8, com.redhat.license_terms=https://centos.org/legal/licensing-policy/) 2024-04-24T13:03:12.635 INFO:journalctl@ceph.mon.smithi114.smithi114.stdout:Apr 24 13:03:12 smithi114 podman[25124]: 2024-04-24 13:03:12.402426318 +0000 UTC m=+0.626363264 container cleanup 73aec54b7d5c8b66d5e4a8d120954329b1c24315a600019adbb88cf8191cd9c5 (image=quay.io/ceph/ceph@sha256:12a0a4f43413fd97a14a3d47a3451b2d2df50020835bb93db666209f3f77617a, name=ceph-9150213e-023a-11ef-bc93-c7b262605968-mon-smithi114, maintainer=Guillaume Abrioux , ceph=True, summary=Provides a CentOS Stream container based on the Red Hat Universal Base Image, com.redhat.build-host=cpt-1002.osbs.prod.upshift.rdu2.redhat.com, description=CentOS Stream is a continuously delivered distro that tracks just ahead of Red Hat Enterprise Linux development. This image takes the Red Hat UBI and layers on content from CentOS Stream, vcs-type=git, com.redhat.component=centos-stream-container, io.buildah.version=1.19.8, GIT_COMMIT=b613db0f44179c0940781c1c7fe04e1acb7093ac, GIT_BRANCH=HEAD, GIT_REPO=https://github.com/ceph/ceph-container.git, CEPH_POINT_RELEASE=-17.2.0, release=754, io.openshift.tags=base centos centos-stream, io.k8s.display-name=CentOS Stream 8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.license_terms=https://centos.org/legal/licensing-policy/, vendor=Red Hat, Inc., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.6-754, version=8, GIT_CLEAN=True, RELEASE=HEAD, build-date=2022-05-03T08:36:31.336870, io.openshift.expose-services=, vcs-ref=f1ee6e37554363ec55e0035aba1a693d3627fdeb, name=centos-stream, architecture=x86_64, distribution-scope=public) 2024-04-24T13:03:12.635 INFO:journalctl@ceph.mon.smithi114.smithi114.stdout:Apr 24 13:03:12 smithi114 bash[25124]: ceph-9150213e-023a-11ef-bc93-c7b262605968-mon-smithi114 2024-04-24T13:03:12.635 INFO:journalctl@ceph.mon.smithi114.smithi114.stdout:Apr 24 13:03:12 smithi114 bash[25151]: Error: no container with name or ID "ceph-9150213e-023a-11ef-bc93-c7b262605968-mon.smithi114" found: no such container 2024-04-24T13:03:12.640 DEBUG:teuthology.orchestra.run.smithi114:> sudo pkill -f 'journalctl -f -n 0 -u ceph-9150213e-023a-11ef-bc93-c7b262605968@mon.smithi114.service' 2024-04-24T13:03:12.692 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-24T13:03:12.692 INFO:tasks.cephadm.mon.smithi114:Stopped mon.smithi114 2024-04-24T13:03:12.693 DEBUG:teuthology.orchestra.run.smithi069:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 9150213e-023a-11ef-bc93-c7b262605968 --force --keep-logs 2024-04-24T13:03:19.901 DEBUG:teuthology.orchestra.run.smithi114:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 9150213e-023a-11ef-bc93-c7b262605968 --force --keep-logs 2024-04-24T13:03:23.187 DEBUG:teuthology.orchestra.run.smithi069:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-04-24T13:03:23.219 DEBUG:teuthology.orchestra.run.smithi114:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-04-24T13:03:23.245 INFO:tasks.cephadm:Archiving crash dumps... 2024-04-24T13:03:23.247 DEBUG:teuthology.misc:Transferring archived files from smithi069:/var/lib/ceph/9150213e-023a-11ef-bc93-c7b262605968/crash to /home/teuthworker/archive/adking-2024-04-24_11:41:41-orch:cephadm-wip-adk-testing-2024-04-23-1222-distro-default-smithi/7671319/remote/smithi069/crash 2024-04-24T13:03:23.248 DEBUG:teuthology.orchestra.run.smithi069:> sudo tar c -f - -C /var/lib/ceph/9150213e-023a-11ef-bc93-c7b262605968/crash -- . 2024-04-24T13:03:23.294 INFO:teuthology.orchestra.run.smithi069.stderr:tar: /var/lib/ceph/9150213e-023a-11ef-bc93-c7b262605968/crash: Cannot open: No such file or directory 2024-04-24T13:03:23.294 INFO:teuthology.orchestra.run.smithi069.stderr:tar: Error is not recoverable: exiting now 2024-04-24T13:03:23.296 DEBUG:teuthology.misc:Transferring archived files from smithi114:/var/lib/ceph/9150213e-023a-11ef-bc93-c7b262605968/crash to /home/teuthworker/archive/adking-2024-04-24_11:41:41-orch:cephadm-wip-adk-testing-2024-04-23-1222-distro-default-smithi/7671319/remote/smithi114/crash 2024-04-24T13:03:23.296 DEBUG:teuthology.orchestra.run.smithi114:> sudo tar c -f - -C /var/lib/ceph/9150213e-023a-11ef-bc93-c7b262605968/crash -- . 2024-04-24T13:03:23.330 INFO:teuthology.orchestra.run.smithi114.stderr:tar: /var/lib/ceph/9150213e-023a-11ef-bc93-c7b262605968/crash: Cannot open: No such file or directory 2024-04-24T13:03:23.330 INFO:teuthology.orchestra.run.smithi114.stderr:tar: Error is not recoverable: exiting now 2024-04-24T13:03:23.331 INFO:tasks.cephadm:Checking cluster log for badness... 2024-04-24T13:03:23.331 DEBUG:teuthology.orchestra.run.smithi069:> sudo egrep '\[ERR\]|\[WRN\]|\[SEC\]' /var/log/ceph/9150213e-023a-11ef-bc93-c7b262605968/ceph.log | egrep CEPHADM_ | egrep -v '\(MDS_ALL_DOWN\)' | egrep -v '\(MDS_UP_LESS_THAN_MAX\)' | head -n 1 2024-04-24T13:03:23.362 INFO:tasks.cephadm:Compressing logs... 2024-04-24T13:03:23.362 DEBUG:teuthology.orchestra.run.smithi069:> time sudo find /var/log/ceph /var/log/rbd-target-api -name '*.log' -print0 | sudo xargs --max-args=1 --max-procs=0 --verbose -0 --no-run-if-empty -- gzip -5 --verbose -- 2024-04-24T13:03:23.405 DEBUG:teuthology.orchestra.run.smithi114:> time sudo find /var/log/ceph /var/log/rbd-target-api -name '*.log' -print0 | sudo xargs --max-args=1 --max-procs=0 --verbose -0 --no-run-if-empty -- gzip -5 --verbose -- 2024-04-24T13:03:23.429 INFO:teuthology.orchestra.run.smithi069.stderr:find: ‘/var/log/rbd-target-api’gzip -5 --verbose --: No such file or directory /var/log/ceph/9150213e-023a-11ef-bc93-c7b262605968/ceph.log 2024-04-24T13:03:23.429 INFO:teuthology.orchestra.run.smithi069.stderr: 2024-04-24T13:03:23.429 INFO:teuthology.orchestra.run.smithi069.stderr:gzip -5 --verbose -- /var/log/ceph/9150213e-023a-11ef-bc93-c7b262605968/ceph-mon.smithi069.log 2024-04-24T13:03:23.429 INFO:teuthology.orchestra.run.smithi069.stderr:gzip -5 --verbose -- /var/log/ceph/9150213e-023a-11ef-bc93-c7b262605968/ceph.audit.log 2024-04-24T13:03:23.429 INFO:teuthology.orchestra.run.smithi069.stderr:/var/log/ceph/9150213e-023a-11ef-bc93-c7b262605968/ceph.log: /var/log/ceph/9150213e-023a-11ef-bc93-c7b262605968/ceph-mon.smithi069.log: gzip -5 --verbose -- /var/log/ceph/9150213e-023a-11ef-bc93-c7b262605968/ceph-mgr.smithi069.vspncx.log 2024-04-24T13:03:23.429 INFO:teuthology.orchestra.run.smithi069.stderr: 84.6% -- replaced with /var/log/ceph/9150213e-023a-11ef-bc93-c7b262605968/ceph.log.gz 2024-04-24T13:03:23.430 INFO:teuthology.orchestra.run.smithi069.stderr:/var/log/ceph/9150213e-023a-11ef-bc93-c7b262605968/ceph.audit.log: gzip -5 --verbose -- /var/log/ceph/9150213e-023a-11ef-bc93-c7b262605968/ceph.cephadm.log 2024-04-24T13:03:23.430 INFO:teuthology.orchestra.run.smithi069.stderr:/var/log/ceph/9150213e-023a-11ef-bc93-c7b262605968/ceph-mgr.smithi069.vspncx.log: gzip -5 --verbose -- /var/log/ceph/9150213e-023a-11ef-bc93-c7b262605968/ceph-volume.log 2024-04-24T13:03:23.430 INFO:teuthology.orchestra.run.smithi069.stderr:/var/log/ceph/9150213e-023a-11ef-bc93-c7b262605968/ceph.cephadm.log: 89.2% -- replaced with /var/log/ceph/9150213e-023a-11ef-bc93-c7b262605968/ceph.audit.log.gz 2024-04-24T13:03:23.430 INFO:teuthology.orchestra.run.smithi069.stderr:gzip -5 --verbose -- /var/log/ceph/cephadm.log 2024-04-24T13:03:23.430 INFO:teuthology.orchestra.run.smithi069.stderr: 81.5% -- replaced with /var/log/ceph/9150213e-023a-11ef-bc93-c7b262605968/ceph.cephadm.log.gz 2024-04-24T13:03:23.430 INFO:teuthology.orchestra.run.smithi069.stderr:/var/log/ceph/9150213e-023a-11ef-bc93-c7b262605968/ceph-volume.log: /var/log/ceph/cephadm.log: 91.1% -- replaced with /var/log/ceph/9150213e-023a-11ef-bc93-c7b262605968/ceph-volume.log.gz 2024-04-24T13:03:23.431 INFO:teuthology.orchestra.run.smithi114.stderr:find: ‘/var/log/rbd-target-api’gzip -5 --verbose -- /var/log/ceph/9150213e-023a-11ef-bc93-c7b262605968/ceph.log 2024-04-24T13:03:23.431 INFO:teuthology.orchestra.run.smithi114.stderr:: No such file or directory 2024-04-24T13:03:23.432 INFO:teuthology.orchestra.run.smithi114.stderr:gzip -5 --verbose -- /var/log/ceph/9150213e-023a-11ef-bc93-c7b262605968/ceph.audit.log 2024-04-24T13:03:23.432 INFO:teuthology.orchestra.run.smithi114.stderr:gzip -5 --verbose -- /var/log/ceph/9150213e-023a-11ef-bc93-c7b262605968/ceph-mgr.smithi114.oaurwi.log 2024-04-24T13:03:23.432 INFO:teuthology.orchestra.run.smithi114.stderr:/var/log/ceph/9150213e-023a-11ef-bc93-c7b262605968/ceph.log: /var/log/ceph/9150213e-023a-11ef-bc93-c7b262605968/ceph.audit.log: gzip -5 --verbose -- /var/log/ceph/9150213e-023a-11ef-bc93-c7b262605968/ceph.cephadm.log 2024-04-24T13:03:23.432 INFO:teuthology.orchestra.run.smithi114.stderr: 77.6% -- replaced with /var/log/ceph/9150213e-023a-11ef-bc93-c7b262605968/ceph.log.gz 2024-04-24T13:03:23.432 INFO:teuthology.orchestra.run.smithi069.stderr: 92.1% -- replaced with /var/log/ceph/cephadm.log.gz 2024-04-24T13:03:23.434 INFO:teuthology.orchestra.run.smithi114.stderr: 87.0% -- replaced with /var/log/ceph/9150213e-023a-11ef-bc93-c7b262605968/ceph.audit.log.gz 2024-04-24T13:03:23.434 INFO:teuthology.orchestra.run.smithi114.stderr:/var/log/ceph/9150213e-023a-11ef-bc93-c7b262605968/ceph-mgr.smithi114.oaurwi.log: gzip -5 --verbose -- /var/log/ceph/9150213e-023a-11ef-bc93-c7b262605968/ceph-volume.log 2024-04-24T13:03:23.434 INFO:teuthology.orchestra.run.smithi114.stderr:/var/log/ceph/9150213e-023a-11ef-bc93-c7b262605968/ceph.cephadm.log: gzip -5 --verbose -- /var/log/ceph/9150213e-023a-11ef-bc93-c7b262605968/ceph-mon.smithi114.log 2024-04-24T13:03:23.434 INFO:teuthology.orchestra.run.smithi114.stderr: 59.4% -- replaced with /var/log/ceph/9150213e-023a-11ef-bc93-c7b262605968/ceph.cephadm.log.gz 2024-04-24T13:03:23.434 INFO:teuthology.orchestra.run.smithi114.stderr:/var/log/ceph/9150213e-023a-11ef-bc93-c7b262605968/ceph-volume.log: gzip -5 --verbose -- /var/log/ceph/cephadm.log 2024-04-24T13:03:23.434 INFO:teuthology.orchestra.run.smithi114.stderr:/var/log/ceph/9150213e-023a-11ef-bc93-c7b262605968/ceph-mon.smithi114.log: 89.3% -- replaced with /var/log/ceph/9150213e-023a-11ef-bc93-c7b262605968/ceph-volume.log.gz 2024-04-24T13:03:23.434 INFO:teuthology.orchestra.run.smithi114.stderr:/var/log/ceph/cephadm.log: 90.2% -- replaced with /var/log/ceph/cephadm.log.gz 2024-04-24T13:03:23.434 INFO:teuthology.orchestra.run.smithi114.stderr: 90.2% -- replaced with /var/log/ceph/9150213e-023a-11ef-bc93-c7b262605968/ceph-mgr.smithi114.oaurwi.log.gz 2024-04-24T13:03:23.435 INFO:teuthology.orchestra.run.smithi114.stderr: 92.6% -- replaced with /var/log/ceph/9150213e-023a-11ef-bc93-c7b262605968/ceph-mon.smithi114.log.gz 2024-04-24T13:03:23.436 INFO:teuthology.orchestra.run.smithi114.stderr: 2024-04-24T13:03:23.436 INFO:teuthology.orchestra.run.smithi114.stderr:real 0m0.017s 2024-04-24T13:03:23.436 INFO:teuthology.orchestra.run.smithi114.stderr:user 0m0.016s 2024-04-24T13:03:23.436 INFO:teuthology.orchestra.run.smithi114.stderr:sys 0m0.016s 2024-04-24T13:03:23.460 INFO:teuthology.orchestra.run.smithi069.stderr: 87.8% -- replaced with /var/log/ceph/9150213e-023a-11ef-bc93-c7b262605968/ceph-mgr.smithi069.vspncx.log.gz 2024-04-24T13:03:23.478 INFO:teuthology.orchestra.run.smithi069.stderr: 91.4% -- replaced with /var/log/ceph/9150213e-023a-11ef-bc93-c7b262605968/ceph-mon.smithi069.log.gz 2024-04-24T13:03:23.480 INFO:teuthology.orchestra.run.smithi069.stderr: 2024-04-24T13:03:23.480 INFO:teuthology.orchestra.run.smithi069.stderr:real 0m0.062s 2024-04-24T13:03:23.480 INFO:teuthology.orchestra.run.smithi069.stderr:user 0m0.096s 2024-04-24T13:03:23.480 INFO:teuthology.orchestra.run.smithi069.stderr:sys 0m0.016s 2024-04-24T13:03:23.480 INFO:tasks.cephadm:Archiving logs... 2024-04-24T13:03:23.481 DEBUG:teuthology.misc:Transferring archived files from smithi069:/var/log/ceph to /home/teuthworker/archive/adking-2024-04-24_11:41:41-orch:cephadm-wip-adk-testing-2024-04-23-1222-distro-default-smithi/7671319/remote/smithi069/log 2024-04-24T13:03:23.482 DEBUG:teuthology.orchestra.run.smithi069:> sudo tar c -f - -C /var/log/ceph -- . 2024-04-24T13:03:23.564 DEBUG:teuthology.misc:Transferring archived files from smithi114:/var/log/ceph to /home/teuthworker/archive/adking-2024-04-24_11:41:41-orch:cephadm-wip-adk-testing-2024-04-23-1222-distro-default-smithi/7671319/remote/smithi114/log 2024-04-24T13:03:23.565 DEBUG:teuthology.orchestra.run.smithi114:> sudo tar c -f - -C /var/log/ceph -- . 2024-04-24T13:03:23.597 INFO:tasks.cephadm:Removing cluster... 2024-04-24T13:03:23.598 DEBUG:teuthology.orchestra.run.smithi069:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 9150213e-023a-11ef-bc93-c7b262605968 --force 2024-04-24T13:03:23.864 DEBUG:teuthology.orchestra.run.smithi114:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 9150213e-023a-11ef-bc93-c7b262605968 --force 2024-04-24T13:03:24.131 INFO:tasks.cephadm:Removing cephadm ... 2024-04-24T13:03:24.131 DEBUG:teuthology.orchestra.run.smithi069:> rm -rf /home/ubuntu/cephtest/cephadm 2024-04-24T13:03:24.145 DEBUG:teuthology.orchestra.run.smithi114:> rm -rf /home/ubuntu/cephtest/cephadm 2024-04-24T13:03:24.160 INFO:tasks.cephadm:Teardown complete 2024-04-24T13:03:24.161 ERROR:teuthology.run_tasks:Saw exception from tasks. Traceback (most recent call last): File "/home/teuthworker/src/git.ceph.com_teuthology_9481b1d62f50e7d0a4f3dd83adf6945b08d5ff17/teuthology/run_tasks.py", line 109, in run_tasks manager.__enter__() File "/usr/lib/python3.8/contextlib.py", line 113, in __enter__ return next(self.gen) File "/home/teuthworker/src/git.ceph.com_ceph-c_23fcfb96e7e1a49d12a94e3f87a8e3f06db2a1ec/qa/tasks/cephadm.py", line 2220, in task with contextutil.nested( File "/usr/lib/python3.8/contextlib.py", line 113, in __enter__ return next(self.gen) File "/home/teuthworker/src/git.ceph.com_teuthology_9481b1d62f50e7d0a4f3dd83adf6945b08d5ff17/teuthology/contextutil.py", line 54, in nested raise exc[1] File "/usr/lib/python3.8/contextlib.py", line 131, in __exit__ self.gen.throw(type, value, traceback) File "/home/teuthworker/src/git.ceph.com_ceph-c_23fcfb96e7e1a49d12a94e3f87a8e3f06db2a1ec/qa/tasks/cephadm.py", line 1849, in initialize_config yield File "/home/teuthworker/src/git.ceph.com_teuthology_9481b1d62f50e7d0a4f3dd83adf6945b08d5ff17/teuthology/contextutil.py", line 30, in nested vars.append(enter()) File "/usr/lib/python3.8/contextlib.py", line 113, in __enter__ return next(self.gen) File "/home/teuthworker/src/git.ceph.com_ceph-c_23fcfb96e7e1a49d12a94e3f87a8e3f06db2a1ec/qa/tasks/cephadm.py", line 1091, in ceph_osds remote.run( File "/home/teuthworker/src/git.ceph.com_teuthology_9481b1d62f50e7d0a4f3dd83adf6945b08d5ff17/teuthology/orchestra/remote.py", line 523, in run r = self._runner(client=self.ssh, name=self.shortname, **kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_9481b1d62f50e7d0a4f3dd83adf6945b08d5ff17/teuthology/orchestra/run.py", line 455, in run r.wait() File "/home/teuthworker/src/git.ceph.com_teuthology_9481b1d62f50e7d0a4f3dd83adf6945b08d5ff17/teuthology/orchestra/run.py", line 161, in wait self._raise_for_status() File "/home/teuthworker/src/git.ceph.com_teuthology_9481b1d62f50e7d0a4f3dd83adf6945b08d5ff17/teuthology/orchestra/run.py", line 181, in _raise_for_status raise CommandFailedError( teuthology.exceptions.CommandFailedError: Command failed on smithi069 with status 1: 'sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9150213e-023a-11ef-bc93-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_4' 2024-04-24T13:03:24.384 ERROR:teuthology.util.sentry: Sentry event: https://sentry.ceph.com/organizations/ceph/?query=0800cdf09648484cad009cf8d1b692a4 Traceback (most recent call last): File "/home/teuthworker/src/git.ceph.com_teuthology_9481b1d62f50e7d0a4f3dd83adf6945b08d5ff17/teuthology/run_tasks.py", line 109, in run_tasks manager.__enter__() File "/usr/lib/python3.8/contextlib.py", line 113, in __enter__ return next(self.gen) File "/home/teuthworker/src/git.ceph.com_ceph-c_23fcfb96e7e1a49d12a94e3f87a8e3f06db2a1ec/qa/tasks/cephadm.py", line 2220, in task with contextutil.nested( File "/usr/lib/python3.8/contextlib.py", line 113, in __enter__ return next(self.gen) File "/home/teuthworker/src/git.ceph.com_teuthology_9481b1d62f50e7d0a4f3dd83adf6945b08d5ff17/teuthology/contextutil.py", line 54, in nested raise exc[1] File "/usr/lib/python3.8/contextlib.py", line 131, in __exit__ self.gen.throw(type, value, traceback) File "/home/teuthworker/src/git.ceph.com_ceph-c_23fcfb96e7e1a49d12a94e3f87a8e3f06db2a1ec/qa/tasks/cephadm.py", line 1849, in initialize_config yield File "/home/teuthworker/src/git.ceph.com_teuthology_9481b1d62f50e7d0a4f3dd83adf6945b08d5ff17/teuthology/contextutil.py", line 30, in nested vars.append(enter()) File "/usr/lib/python3.8/contextlib.py", line 113, in __enter__ return next(self.gen) File "/home/teuthworker/src/git.ceph.com_ceph-c_23fcfb96e7e1a49d12a94e3f87a8e3f06db2a1ec/qa/tasks/cephadm.py", line 1091, in ceph_osds remote.run( File "/home/teuthworker/src/git.ceph.com_teuthology_9481b1d62f50e7d0a4f3dd83adf6945b08d5ff17/teuthology/orchestra/remote.py", line 523, in run r = self._runner(client=self.ssh, name=self.shortname, **kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_9481b1d62f50e7d0a4f3dd83adf6945b08d5ff17/teuthology/orchestra/run.py", line 455, in run r.wait() File "/home/teuthworker/src/git.ceph.com_teuthology_9481b1d62f50e7d0a4f3dd83adf6945b08d5ff17/teuthology/orchestra/run.py", line 161, in wait self._raise_for_status() File "/home/teuthworker/src/git.ceph.com_teuthology_9481b1d62f50e7d0a4f3dd83adf6945b08d5ff17/teuthology/orchestra/run.py", line 181, in _raise_for_status raise CommandFailedError( teuthology.exceptions.CommandFailedError: Command failed on smithi069 with status 1: 'sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9150213e-023a-11ef-bc93-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_4' 2024-04-24T13:03:24.388 DEBUG:teuthology.run_tasks:Unwinding manager cephadm 2024-04-24T13:03:24.397 DEBUG:teuthology.run_tasks:Unwinding manager clock 2024-04-24T13:03:24.410 INFO:teuthology.task.clock:Checking final clock skew... 2024-04-24T13:03:24.411 DEBUG:teuthology.orchestra.run.smithi069:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-04-24T13:03:24.425 DEBUG:teuthology.orchestra.run.smithi114:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-04-24T13:03:24.430 INFO:teuthology.orchestra.run.smithi069.stderr:bash: line 1: ntpq: command not found 2024-04-24T13:03:24.433 INFO:teuthology.orchestra.run.smithi069.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-04-24T13:03:24.433 INFO:teuthology.orchestra.run.smithi069.stdout:=============================================================================== 2024-04-24T13:03:24.433 INFO:teuthology.orchestra.run.smithi069.stdout:^- hv01.front.sepia.ceph.com 2 6 177 0 +282us[ +282us] +/- 590ms 2024-04-24T13:03:24.434 INFO:teuthology.orchestra.run.smithi069.stdout:^* hv02.front.sepia.ceph.com 3 6 177 1 -556us[ -559us] +/- 72ms 2024-04-24T13:03:24.434 INFO:teuthology.orchestra.run.smithi069.stdout:^+ hv03.front.sepia.ceph.com 3 6 77 64 +657us[ +656us] +/- 85ms 2024-04-24T13:03:24.434 INFO:teuthology.orchestra.run.smithi069.stdout:^? hv04.front.sepia.ceph.com 0 8 0 - +0ns[ +0ns] +/- 0ns 2024-04-24T13:03:24.440 INFO:teuthology.orchestra.run.smithi114.stderr:bash: line 1: ntpq: command not found 2024-04-24T13:03:24.445 INFO:teuthology.orchestra.run.smithi114.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-04-24T13:03:24.446 INFO:teuthology.orchestra.run.smithi114.stdout:=============================================================================== 2024-04-24T13:03:24.446 INFO:teuthology.orchestra.run.smithi114.stdout:^- hv01.front.sepia.ceph.com 2 6 177 0 +284us[ +284us] +/- 590ms 2024-04-24T13:03:24.446 INFO:teuthology.orchestra.run.smithi114.stdout:^* hv02.front.sepia.ceph.com 3 6 177 1 -543us[ -535us] +/- 72ms 2024-04-24T13:03:24.446 INFO:teuthology.orchestra.run.smithi114.stdout:^+ hv03.front.sepia.ceph.com 3 6 177 2 +652us[ +660us] +/- 85ms 2024-04-24T13:03:24.446 INFO:teuthology.orchestra.run.smithi114.stdout:^? hv04.front.sepia.ceph.com 0 8 0 - +0ns[ +0ns] +/- 0ns 2024-04-24T13:03:24.447 DEBUG:teuthology.run_tasks:Unwinding manager ansible.cephlab 2024-04-24T13:03:24.455 INFO:teuthology.task.ansible:Skipping ansible cleanup... 2024-04-24T13:03:24.456 DEBUG:teuthology.run_tasks:Unwinding manager selinux 2024-04-24T13:03:24.465 DEBUG:teuthology.orchestra.run.smithi069:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2024-04-24T13:03:24.533 DEBUG:teuthology.orchestra.run.smithi114:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2024-04-24T13:03:24.590 DEBUG:teuthology.orchestra.run.smithi069:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 -e scontext=system_u:system_r:getty_t:s0 2024-04-24T13:03:24.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-24T13:03:24.616 DEBUG:teuthology.orchestra.run.smithi114:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 -e scontext=system_u:system_r:getty_t:s0 2024-04-24T13:03:24.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-24T13:03:24.642 DEBUG:teuthology.run_tasks:Unwinding manager pcp 2024-04-24T13:03:24.651 DEBUG:teuthology.run_tasks:Unwinding manager internal.timer 2024-04-24T13:03:24.661 INFO:teuthology.task.internal:Duration was 412.172058 seconds 2024-04-24T13:03:24.662 DEBUG:teuthology.run_tasks:Unwinding manager internal.syslog 2024-04-24T13:03:24.671 INFO:teuthology.task.internal.syslog:Shutting down syslog monitoring... 2024-04-24T13:03:24.671 DEBUG:teuthology.orchestra.run.smithi069:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2024-04-24T13:03:24.674 DEBUG:teuthology.orchestra.run.smithi114:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2024-04-24T13:03:24.707 INFO:teuthology.orchestra.run.smithi069.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-04-24T13:03:24.718 INFO:teuthology.orchestra.run.smithi114.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-04-24T13:03:24.981 INFO:teuthology.task.internal.syslog:Checking logs for errors... 2024-04-24T13:03:24.981 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi069.front.sepia.ceph.com 2024-04-24T13:03:24.982 DEBUG:teuthology.orchestra.run.smithi069:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2024-04-24T13:03:25.001 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi114.front.sepia.ceph.com 2024-04-24T13:03:25.002 DEBUG:teuthology.orchestra.run.smithi114:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2024-04-24T13:03:25.048 INFO:teuthology.task.internal.syslog:Compressing syslogs... 2024-04-24T13:03:25.049 DEBUG:teuthology.orchestra.run.smithi069:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2024-04-24T13:03:25.052 DEBUG:teuthology.orchestra.run.smithi114:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2024-04-24T13:03:25.113 INFO:teuthology.task.internal.syslog:Gathering journactl -b0... 2024-04-24T13:03:25.114 DEBUG:teuthology.orchestra.run.smithi069:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2024-04-24T13:03:25.117 DEBUG:teuthology.orchestra.run.smithi114:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2024-04-24T13:03:25.266 DEBUG:teuthology.run_tasks:Unwinding manager internal.sudo 2024-04-24T13:03:25.275 INFO:teuthology.task.internal:Restoring /etc/sudoers... 2024-04-24T13:03:25.276 DEBUG:teuthology.orchestra.run.smithi069:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2024-04-24T13:03:25.299 DEBUG:teuthology.orchestra.run.smithi114:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2024-04-24T13:03:25.322 DEBUG:teuthology.run_tasks:Unwinding manager internal.coredump 2024-04-24T13:03:25.333 DEBUG:teuthology.orchestra.run.smithi069:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2024-04-24T13:03:25.342 DEBUG:teuthology.orchestra.run.smithi114:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2024-04-24T13:03:25.361 INFO:teuthology.orchestra.run.smithi069.stdout:kernel.core_pattern = core 2024-04-24T13:03:25.385 INFO:teuthology.orchestra.run.smithi114.stdout:kernel.core_pattern = core 2024-04-24T13:03:25.401 DEBUG:teuthology.orchestra.run.smithi069:> test -e /home/ubuntu/cephtest/archive/coredump 2024-04-24T13:03:25.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-24T13:03:25.436 DEBUG:teuthology.orchestra.run.smithi114:> test -e /home/ubuntu/cephtest/archive/coredump 2024-04-24T13:03:25.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-24T13:03:25.455 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive 2024-04-24T13:03:25.466 INFO:teuthology.task.internal:Transferring archived files... 2024-04-24T13:03:25.467 DEBUG:teuthology.misc:Transferring archived files from smithi069:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/adking-2024-04-24_11:41:41-orch:cephadm-wip-adk-testing-2024-04-23-1222-distro-default-smithi/7671319/remote/smithi069 2024-04-24T13:03:25.468 DEBUG:teuthology.orchestra.run.smithi069:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2024-04-24T13:03:25.511 DEBUG:teuthology.misc:Transferring archived files from smithi114:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/adking-2024-04-24_11:41:41-orch:cephadm-wip-adk-testing-2024-04-23-1222-distro-default-smithi/7671319/remote/smithi114 2024-04-24T13:03:25.511 DEBUG:teuthology.orchestra.run.smithi114:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2024-04-24T13:03:25.542 INFO:teuthology.task.internal:Removing archive directory... 2024-04-24T13:03:25.543 DEBUG:teuthology.orchestra.run.smithi069:> rm -rf -- /home/ubuntu/cephtest/archive 2024-04-24T13:03:25.547 DEBUG:teuthology.orchestra.run.smithi114:> rm -rf -- /home/ubuntu/cephtest/archive 2024-04-24T13:03:25.592 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive_upload 2024-04-24T13:03:25.603 INFO:teuthology.task.internal:Not uploading archives. 2024-04-24T13:03:25.603 DEBUG:teuthology.run_tasks:Unwinding manager internal.base 2024-04-24T13:03:25.614 INFO:teuthology.task.internal:Tidying up after the test... 2024-04-24T13:03:25.614 DEBUG:teuthology.orchestra.run.smithi069:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2024-04-24T13:03:25.617 DEBUG:teuthology.orchestra.run.smithi114:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2024-04-24T13:03:25.629 INFO:teuthology.orchestra.run.smithi069.stdout: 262213 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Apr 24 13:03 /home/ubuntu/cephtest 2024-04-24T13:03:25.646 INFO:teuthology.orchestra.run.smithi114.stdout: 262213 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Apr 24 13:03 /home/ubuntu/cephtest 2024-04-24T13:03:25.647 DEBUG:teuthology.run_tasks:Unwinding manager kernel 2024-04-24T13:03:25.657 DEBUG:teuthology.run_tasks:Unwinding manager console_log 2024-04-24T13:03:25.679 INFO:teuthology.nuke:Checking targets against current locks 2024-04-24T13:03:25.707 DEBUG:teuthology.nuke:shortname: smithi069 2024-04-24T13:03:25.707 INFO:teuthology.task.internal.check_lock:Checking locks... 2024-04-24T13:03:25.735 DEBUG:teuthology.nuke:shortname: smithi114 2024-04-24T13:03:25.735 INFO:teuthology.task.internal.check_lock:Checking locks... 2024-04-24T13:03:25.740 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi069.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/adking-2024-04-24_11:41:41-orch:cephadm-wip-adk-testing-2024-04-23-1222-distro-default-smithi/7671319', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-04-24 12:50:59.575279', 'locked_by': 'scheduled_adking@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBH/Hfuo3Uojpx7SKQktHR32R8HvKf9OuIXSZ8JIl9/E/4Bd6qqiOIUd9ftefIKFCMsiPPA2UdaPElPjjs5W5bS0='} 2024-04-24T13:03:25.760 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi114.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/adking-2024-04-24_11:41:41-orch:cephadm-wip-adk-testing-2024-04-23-1222-distro-default-smithi/7671319', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-04-24 12:50:59.573290', 'locked_by': 'scheduled_adking@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBELW/agFKO94t3difRrRkj0O1Nn8I7WShXt1NPx7P+DTz41jSKcynLn4XKES8hblVhzzf3zZ8BmrnMB53qZI7gU='} 2024-04-24T13:03:25.805 INFO:teuthology.orchestra.console.smithi069:Power off 2024-04-24T13:03:25.805 DEBUG:teuthology.orchestra.console.smithi069:pexpect command: ipmitool -H smithi069.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power off 2024-04-24T13:03:25.823 INFO:teuthology.orchestra.console.smithi114:Power off 2024-04-24T13:03:25.824 DEBUG:teuthology.orchestra.console.smithi114:pexpect command: ipmitool -H smithi114.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power off 2024-04-24T13:03:25.834 DEBUG:teuthology.orchestra.console.smithi069:power off output: Chassis Power Control: Down/Off 2024-04-24T13:03:25.835 DEBUG:teuthology.orchestra.console.smithi069:pexpect command: ipmitool -H smithi069.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-24T13:03:25.847 DEBUG:teuthology.orchestra.console.smithi114:power off output: Chassis Power Control: Down/Off 2024-04-24T13:03:25.848 DEBUG:teuthology.orchestra.console.smithi114:pexpect command: ipmitool -H smithi114.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-24T13:03:25.862 DEBUG:teuthology.orchestra.console.smithi069:check power output: Chassis Power is on 2024-04-24T13:03:25.871 DEBUG:teuthology.orchestra.console.smithi114:check power output: Chassis Power is on 2024-04-24T13:03:29.863 DEBUG:teuthology.orchestra.console.smithi069:pexpect command: ipmitool -H smithi069.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-24T13:03:29.874 DEBUG:teuthology.orchestra.console.smithi114:pexpect command: ipmitool -H smithi114.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-24T13:03:29.977 DEBUG:teuthology.orchestra.console.smithi069:check power output: Chassis Power is on 2024-04-24T13:03:29.986 DEBUG:teuthology.orchestra.console.smithi114:check power output: Chassis Power is on 2024-04-24T13:03:33.978 DEBUG:teuthology.orchestra.console.smithi069:pexpect command: ipmitool -H smithi069.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-24T13:03:33.987 DEBUG:teuthology.orchestra.console.smithi114:pexpect command: ipmitool -H smithi114.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-24T13:03:34.089 DEBUG:teuthology.orchestra.console.smithi069:check power output: Chassis Power is off 2024-04-24T13:03:34.097 DEBUG:teuthology.orchestra.console.smithi114:check power output: Chassis Power is on 2024-04-24T13:03:34.191 INFO:teuthology.orchestra.console.smithi069:Power off completed 2024-04-24T13:03:38.098 DEBUG:teuthology.orchestra.console.smithi114:pexpect command: ipmitool -H smithi114.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-24T13:03:38.211 DEBUG:teuthology.orchestra.console.smithi114:check power output: Chassis Power is off 2024-04-24T13:03:38.312 INFO:teuthology.orchestra.console.smithi114:Power off completed 2024-04-24T13:03:38.419 INFO:teuthology.run:Summary data: description: orch:cephadm/mgr-nfs-upgrade/{0-centos_9.stream 1-bootstrap/17.2.0 1-start 2-nfs 3-upgrade-with-workload 4-final} duration: 412.17205834388733 failure_reason: 'Command failed on smithi069 with status 1: ''sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9150213e-023a-11ef-bc93-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_4''' owner: scheduled_adking@teuthology sentry_event: https://sentry.ceph.com/organizations/ceph/?query=0800cdf09648484cad009cf8d1b692a4 status: fail success: false 2024-04-24T13:03:38.419 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2024-04-24T13:03:38.503 INFO:teuthology.run:FAIL